{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Use VPG with Baseline to Play CartPole-v0\n",
    "\n",
    "TensorFlow version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import sys\n",
    "import logging\n",
    "import imp\n",
    "import itertools\n",
    "\n",
    "import numpy as np\n",
    "np.random.seed(0)\n",
    "import pandas as pd\n",
    "import gym\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow.compat.v2 as tf\n",
    "tf.random.set_seed(0)\n",
    "from tensorflow import keras\n",
    "from tensorflow import nn\n",
    "from tensorflow import optimizers\n",
    "from tensorflow import losses\n",
    "from tensorflow.keras import layers\n",
    "\n",
    "imp.reload(logging)\n",
    "logging.basicConfig(level=logging.DEBUG,\n",
    "        format='%(asctime)s [%(levelname)s] %(message)s',\n",
    "        stream=sys.stdout, datefmt='%H:%M:%S')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10:46:49 [INFO] env: <CartPoleEnv<CartPole-v0>>\n",
      "10:46:49 [INFO] action_space: Discrete(2)\n",
      "10:46:49 [INFO] observation_space: Box(-3.4028234663852886e+38, 3.4028234663852886e+38, (4,), float32)\n",
      "10:46:49 [INFO] reward_range: (-inf, inf)\n",
      "10:46:49 [INFO] metadata: {'render.modes': ['human', 'rgb_array'], 'video.frames_per_second': 50}\n",
      "10:46:49 [INFO] _max_episode_steps: 200\n",
      "10:46:49 [INFO] _elapsed_steps: None\n"
     ]
    }
   ],
   "source": [
    "env = gym.make('CartPole-v0')\n",
    "env.seed(0)\n",
    "for key in vars(env):\n",
    "    logging.info('%s: %s', key, vars(env)[key])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class VPGwBaselineAgent:\n",
    "    def __init__(self, env):\n",
    "        self.action_n = env.action_space.n\n",
    "        self.gamma = 0.99\n",
    "\n",
    "        self.trajectory = []\n",
    "\n",
    "        self.policy_net = self.build_net(hidden_sizes=[],\n",
    "                output_size=self.action_n,\n",
    "                output_activation=nn.softmax,\n",
    "                loss=losses.categorical_crossentropy,\n",
    "                learning_rate=0.005)\n",
    "        self.baseline_net = self.build_net(hidden_sizes=[],\n",
    "                learning_rate=0.01)\n",
    "\n",
    "    def build_net(self, hidden_sizes, output_size=1,\n",
    "            activation=nn.relu, output_activation=None,\n",
    "            use_bias=False, loss=losses.mse, learning_rate=0.005):\n",
    "        model = keras.Sequential()\n",
    "        for hidden_size in hidden_sizes:\n",
    "            model.add(layers.Dense(units=hidden_size,\n",
    "                    activation=activation, use_bias=use_bias))\n",
    "        model.add(layers.Dense(units=output_size,\n",
    "                activation=output_activation, use_bias=use_bias))\n",
    "        optimizer = optimizers.Adam(learning_rate)\n",
    "        model.compile(optimizer=optimizer, loss=loss)\n",
    "        return model\n",
    "\n",
    "    def reset(self, mode=None):\n",
    "        self.mode = mode\n",
    "        if self.mode == 'train':\n",
    "            self.trajectory = []\n",
    "\n",
    "    def step(self, observation, reward, done):\n",
    "        probs = self.policy_net.predict(observation[np.newaxis])[0]\n",
    "        action = np.random.choice(self.action_n, p=probs)\n",
    "        if self.mode == 'train':\n",
    "            self.trajectory += [observation, reward, done, action]\n",
    "        return action\n",
    "\n",
    "    def close(self):\n",
    "        if self.mode == 'train':\n",
    "            self.learn()\n",
    "\n",
    "    def learn(self):\n",
    "        df = pd.DataFrame(np.array(self.trajectory, dtype=object).reshape(-1, 4),\n",
    "                columns=['state', 'reward', 'done', 'action'])\n",
    "\n",
    "        # train baseline\n",
    "        df['discount'] = self.gamma ** df.index.to_series()\n",
    "        df['discounted_reward'] = df['discount'] * df['reward'].astype(float)\n",
    "        df['discounted_return'] = df['discounted_reward'][::-1].cumsum()\n",
    "        df['return'] = df['discounted_return'] / df['discount']\n",
    "        states = np.stack(df['state'])\n",
    "        returns = df[['return',]].values\n",
    "        self.baseline_net.fit(states, returns, verbose=0)\n",
    "\n",
    "        # train policy\n",
    "        df['baseline'] = self.baseline_net.predict(states)\n",
    "        df['psi'] = df['discounted_return'] - df['baseline'] * df['discount']\n",
    "        actions = np.eye(self.action_n)[df['action'].astype(int)]\n",
    "        sample_weight = df[['discounted_return',]].values\n",
    "        self.policy_net.fit(states, actions, sample_weight=sample_weight, verbose=0)\n",
    "\n",
    "\n",
    "agent = VPGwBaselineAgent(env)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10:46:50 [INFO] ==== train ====\n",
      "10:46:53 [DEBUG] train episode 0: reward = 29.00, steps = 29\n",
      "10:46:53 [INFO] NumExpr defaulting to 8 threads.\n",
      "10:46:59 [DEBUG] train episode 1: reward = 39.00, steps = 39\n",
      "10:47:03 [DEBUG] train episode 2: reward = 44.00, steps = 44\n",
      "10:47:08 [DEBUG] train episode 3: reward = 38.00, steps = 38\n",
      "10:47:11 [DEBUG] train episode 4: reward = 33.00, steps = 33\n",
      "10:47:13 [DEBUG] train episode 5: reward = 20.00, steps = 20\n",
      "10:47:18 [DEBUG] train episode 6: reward = 49.00, steps = 49\n",
      "10:47:24 [DEBUG] train episode 7: reward = 56.00, steps = 56\n",
      "10:47:27 [DEBUG] train episode 8: reward = 30.00, steps = 30\n",
      "10:47:31 [DEBUG] train episode 9: reward = 44.00, steps = 44\n",
      "10:47:34 [DEBUG] train episode 10: reward = 26.00, steps = 26\n",
      "10:47:40 [DEBUG] train episode 11: reward = 64.00, steps = 64\n",
      "10:47:46 [DEBUG] train episode 12: reward = 56.00, steps = 56\n",
      "10:47:48 [DEBUG] train episode 13: reward = 24.00, steps = 24\n",
      "10:47:55 [DEBUG] train episode 14: reward = 65.00, steps = 65\n",
      "10:47:56 [DEBUG] train episode 15: reward = 14.00, steps = 14\n",
      "10:48:01 [DEBUG] train episode 16: reward = 44.00, steps = 44\n",
      "10:48:03 [DEBUG] train episode 17: reward = 20.00, steps = 20\n",
      "10:48:07 [DEBUG] train episode 18: reward = 39.00, steps = 39\n",
      "10:48:11 [DEBUG] train episode 19: reward = 47.00, steps = 47\n",
      "10:48:18 [DEBUG] train episode 20: reward = 61.00, steps = 61\n",
      "10:48:21 [DEBUG] train episode 21: reward = 38.00, steps = 38\n",
      "10:48:23 [DEBUG] train episode 22: reward = 19.00, steps = 19\n",
      "10:48:28 [DEBUG] train episode 23: reward = 40.00, steps = 40\n",
      "10:48:32 [DEBUG] train episode 24: reward = 39.00, steps = 39\n",
      "10:48:38 [DEBUG] train episode 25: reward = 64.00, steps = 64\n",
      "10:48:43 [DEBUG] train episode 26: reward = 47.00, steps = 47\n",
      "10:48:44 [DEBUG] train episode 27: reward = 15.00, steps = 15\n",
      "10:48:53 [DEBUG] train episode 28: reward = 94.00, steps = 94\n",
      "10:48:56 [DEBUG] train episode 29: reward = 19.00, steps = 19\n",
      "10:48:58 [DEBUG] train episode 30: reward = 22.00, steps = 22\n",
      "10:49:03 [DEBUG] train episode 31: reward = 57.00, steps = 57\n",
      "10:49:08 [DEBUG] train episode 32: reward = 50.00, steps = 50\n",
      "10:49:12 [DEBUG] train episode 33: reward = 31.00, steps = 31\n",
      "10:49:15 [DEBUG] train episode 34: reward = 23.00, steps = 23\n",
      "10:49:17 [DEBUG] train episode 35: reward = 26.00, steps = 26\n",
      "10:49:24 [DEBUG] train episode 36: reward = 73.00, steps = 73\n",
      "10:49:27 [DEBUG] train episode 37: reward = 25.00, steps = 25\n",
      "10:49:34 [DEBUG] train episode 38: reward = 67.00, steps = 67\n",
      "10:49:38 [DEBUG] train episode 39: reward = 46.00, steps = 46\n",
      "10:49:43 [DEBUG] train episode 40: reward = 54.00, steps = 54\n",
      "10:49:50 [DEBUG] train episode 41: reward = 66.00, steps = 66\n",
      "10:49:53 [DEBUG] train episode 42: reward = 30.00, steps = 30\n",
      "10:49:58 [DEBUG] train episode 43: reward = 52.00, steps = 52\n",
      "10:50:01 [DEBUG] train episode 44: reward = 26.00, steps = 26\n",
      "10:50:03 [DEBUG] train episode 45: reward = 22.00, steps = 22\n",
      "10:50:06 [DEBUG] train episode 46: reward = 27.00, steps = 27\n",
      "10:50:08 [DEBUG] train episode 47: reward = 16.00, steps = 16\n",
      "10:50:10 [DEBUG] train episode 48: reward = 18.00, steps = 18\n",
      "10:50:12 [DEBUG] train episode 49: reward = 22.00, steps = 22\n",
      "10:50:20 [DEBUG] train episode 50: reward = 76.00, steps = 76\n",
      "10:50:25 [DEBUG] train episode 51: reward = 58.00, steps = 58\n",
      "10:50:30 [DEBUG] train episode 52: reward = 41.00, steps = 41\n",
      "10:50:34 [DEBUG] train episode 53: reward = 47.00, steps = 47\n",
      "10:50:38 [DEBUG] train episode 54: reward = 41.00, steps = 41\n",
      "10:50:45 [DEBUG] train episode 55: reward = 63.00, steps = 63\n",
      "10:50:48 [DEBUG] train episode 56: reward = 26.00, steps = 26\n",
      "10:50:57 [DEBUG] train episode 57: reward = 93.00, steps = 93\n",
      "10:51:02 [DEBUG] train episode 58: reward = 43.00, steps = 43\n",
      "10:51:10 [DEBUG] train episode 59: reward = 83.00, steps = 83\n",
      "10:51:11 [DEBUG] train episode 60: reward = 14.00, steps = 14\n",
      "10:51:15 [DEBUG] train episode 61: reward = 29.00, steps = 29\n",
      "10:51:18 [DEBUG] train episode 62: reward = 37.00, steps = 37\n",
      "10:51:26 [DEBUG] train episode 63: reward = 76.00, steps = 76\n",
      "10:51:28 [DEBUG] train episode 64: reward = 22.00, steps = 22\n",
      "10:51:35 [DEBUG] train episode 65: reward = 78.00, steps = 78\n",
      "10:51:41 [DEBUG] train episode 66: reward = 54.00, steps = 54\n",
      "10:51:45 [DEBUG] train episode 67: reward = 45.00, steps = 45\n",
      "10:51:47 [DEBUG] train episode 68: reward = 13.00, steps = 13\n",
      "10:51:53 [DEBUG] train episode 69: reward = 68.00, steps = 68\n",
      "10:51:59 [DEBUG] train episode 70: reward = 60.00, steps = 60\n",
      "10:52:06 [DEBUG] train episode 71: reward = 76.00, steps = 76\n",
      "10:52:14 [DEBUG] train episode 72: reward = 86.00, steps = 86\n",
      "10:52:20 [DEBUG] train episode 73: reward = 65.00, steps = 65\n",
      "10:52:23 [DEBUG] train episode 74: reward = 18.00, steps = 18\n",
      "10:52:29 [DEBUG] train episode 75: reward = 57.00, steps = 57\n",
      "10:52:32 [DEBUG] train episode 76: reward = 26.00, steps = 26\n",
      "10:52:34 [DEBUG] train episode 77: reward = 22.00, steps = 22\n",
      "10:52:42 [DEBUG] train episode 78: reward = 71.00, steps = 71\n",
      "10:52:50 [DEBUG] train episode 79: reward = 90.00, steps = 90\n",
      "10:52:55 [DEBUG] train episode 80: reward = 45.00, steps = 45\n",
      "10:53:00 [DEBUG] train episode 81: reward = 50.00, steps = 50\n",
      "10:53:03 [DEBUG] train episode 82: reward = 26.00, steps = 26\n",
      "10:53:05 [DEBUG] train episode 83: reward = 14.00, steps = 14\n",
      "10:53:10 [DEBUG] train episode 84: reward = 46.00, steps = 46\n",
      "10:53:15 [DEBUG] train episode 85: reward = 50.00, steps = 50\n",
      "10:53:18 [DEBUG] train episode 86: reward = 23.00, steps = 23\n",
      "10:53:23 [DEBUG] train episode 87: reward = 46.00, steps = 46\n",
      "10:53:27 [DEBUG] train episode 88: reward = 37.00, steps = 37\n",
      "10:53:29 [DEBUG] train episode 89: reward = 22.00, steps = 22\n",
      "10:53:38 [DEBUG] train episode 90: reward = 80.00, steps = 80\n",
      "10:53:43 [DEBUG] train episode 91: reward = 52.00, steps = 52\n",
      "10:53:53 [DEBUG] train episode 92: reward = 98.00, steps = 98\n",
      "10:53:56 [DEBUG] train episode 93: reward = 35.00, steps = 35\n",
      "10:53:59 [DEBUG] train episode 94: reward = 24.00, steps = 24\n",
      "10:54:09 [DEBUG] train episode 95: reward = 95.00, steps = 95\n",
      "10:54:13 [DEBUG] train episode 96: reward = 38.00, steps = 38\n",
      "10:54:18 [DEBUG] train episode 97: reward = 55.00, steps = 55\n",
      "10:54:23 [DEBUG] train episode 98: reward = 42.00, steps = 42\n",
      "10:54:28 [DEBUG] train episode 99: reward = 50.00, steps = 50\n",
      "10:54:37 [DEBUG] train episode 100: reward = 86.00, steps = 86\n",
      "10:54:42 [DEBUG] train episode 101: reward = 45.00, steps = 45\n",
      "10:54:45 [DEBUG] train episode 102: reward = 31.00, steps = 31\n",
      "10:54:49 [DEBUG] train episode 103: reward = 30.00, steps = 30\n",
      "10:54:51 [DEBUG] train episode 104: reward = 19.00, steps = 19\n",
      "10:54:55 [DEBUG] train episode 105: reward = 34.00, steps = 34\n",
      "10:55:02 [DEBUG] train episode 106: reward = 57.00, steps = 57\n",
      "10:55:04 [DEBUG] train episode 107: reward = 19.00, steps = 19\n",
      "10:55:10 [DEBUG] train episode 108: reward = 52.00, steps = 52\n",
      "10:55:13 [DEBUG] train episode 109: reward = 35.00, steps = 35\n",
      "10:55:15 [DEBUG] train episode 110: reward = 18.00, steps = 18\n",
      "10:55:26 [DEBUG] train episode 111: reward = 118.00, steps = 118\n",
      "10:55:30 [DEBUG] train episode 112: reward = 45.00, steps = 45\n",
      "10:55:34 [DEBUG] train episode 113: reward = 35.00, steps = 35\n",
      "10:55:41 [DEBUG] train episode 114: reward = 69.00, steps = 69\n",
      "10:55:44 [DEBUG] train episode 115: reward = 39.00, steps = 39\n",
      "10:55:55 [DEBUG] train episode 116: reward = 114.00, steps = 114\n",
      "10:55:57 [DEBUG] train episode 117: reward = 18.00, steps = 18\n",
      "10:56:03 [DEBUG] train episode 118: reward = 55.00, steps = 55\n",
      "10:56:05 [DEBUG] train episode 119: reward = 29.00, steps = 29\n",
      "10:56:07 [DEBUG] train episode 120: reward = 15.00, steps = 15\n",
      "10:56:12 [DEBUG] train episode 121: reward = 45.00, steps = 45\n",
      "10:56:14 [DEBUG] train episode 122: reward = 21.00, steps = 21\n",
      "10:56:19 [DEBUG] train episode 123: reward = 57.00, steps = 57\n",
      "10:56:24 [DEBUG] train episode 124: reward = 44.00, steps = 44\n",
      "10:56:31 [DEBUG] train episode 125: reward = 71.00, steps = 71\n",
      "10:56:34 [DEBUG] train episode 126: reward = 38.00, steps = 38\n",
      "10:56:41 [DEBUG] train episode 127: reward = 65.00, steps = 65\n",
      "10:56:45 [DEBUG] train episode 128: reward = 34.00, steps = 34\n",
      "10:56:48 [DEBUG] train episode 129: reward = 29.00, steps = 29\n",
      "10:56:51 [DEBUG] train episode 130: reward = 36.00, steps = 36\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10:56:56 [DEBUG] train episode 131: reward = 50.00, steps = 50\n",
      "10:57:01 [DEBUG] train episode 132: reward = 53.00, steps = 53\n",
      "10:57:07 [DEBUG] train episode 133: reward = 54.00, steps = 54\n",
      "10:57:10 [DEBUG] train episode 134: reward = 30.00, steps = 30\n",
      "10:57:18 [DEBUG] train episode 135: reward = 80.00, steps = 80\n",
      "10:57:29 [DEBUG] train episode 136: reward = 113.00, steps = 113\n",
      "10:57:40 [DEBUG] train episode 137: reward = 108.00, steps = 108\n",
      "10:57:45 [DEBUG] train episode 138: reward = 60.00, steps = 60\n",
      "10:57:51 [DEBUG] train episode 139: reward = 54.00, steps = 54\n",
      "10:57:55 [DEBUG] train episode 140: reward = 36.00, steps = 36\n",
      "10:57:58 [DEBUG] train episode 141: reward = 35.00, steps = 35\n",
      "10:58:03 [DEBUG] train episode 142: reward = 42.00, steps = 42\n",
      "10:58:04 [DEBUG] train episode 143: reward = 11.00, steps = 11\n",
      "10:58:07 [DEBUG] train episode 144: reward = 30.00, steps = 30\n",
      "10:58:17 [DEBUG] train episode 145: reward = 94.00, steps = 94\n",
      "10:58:19 [DEBUG] train episode 146: reward = 15.00, steps = 15\n",
      "10:58:22 [DEBUG] train episode 147: reward = 40.00, steps = 40\n",
      "10:58:26 [DEBUG] train episode 148: reward = 30.00, steps = 30\n",
      "10:58:31 [DEBUG] train episode 149: reward = 60.00, steps = 60\n",
      "10:58:35 [DEBUG] train episode 150: reward = 30.00, steps = 30\n",
      "10:58:38 [DEBUG] train episode 151: reward = 30.00, steps = 30\n",
      "10:58:40 [DEBUG] train episode 152: reward = 19.00, steps = 19\n",
      "10:58:48 [DEBUG] train episode 153: reward = 88.00, steps = 88\n",
      "10:58:55 [DEBUG] train episode 154: reward = 67.00, steps = 67\n",
      "10:59:04 [DEBUG] train episode 155: reward = 101.00, steps = 101\n",
      "10:59:06 [DEBUG] train episode 156: reward = 21.00, steps = 21\n",
      "10:59:18 [DEBUG] train episode 157: reward = 129.00, steps = 129\n",
      "10:59:26 [DEBUG] train episode 158: reward = 80.00, steps = 80\n",
      "10:59:30 [DEBUG] train episode 159: reward = 43.00, steps = 43\n",
      "10:59:37 [DEBUG] train episode 160: reward = 73.00, steps = 73\n",
      "10:59:41 [DEBUG] train episode 161: reward = 40.00, steps = 40\n",
      "10:59:47 [DEBUG] train episode 162: reward = 58.00, steps = 58\n",
      "10:59:50 [DEBUG] train episode 163: reward = 34.00, steps = 34\n",
      "10:59:55 [DEBUG] train episode 164: reward = 47.00, steps = 47\n",
      "11:00:01 [DEBUG] train episode 165: reward = 63.00, steps = 63\n",
      "11:00:13 [DEBUG] train episode 166: reward = 125.00, steps = 125\n",
      "11:00:19 [DEBUG] train episode 167: reward = 53.00, steps = 53\n",
      "11:00:25 [DEBUG] train episode 168: reward = 66.00, steps = 66\n",
      "11:00:32 [DEBUG] train episode 169: reward = 72.00, steps = 72\n",
      "11:00:46 [DEBUG] train episode 170: reward = 155.00, steps = 155\n",
      "11:00:49 [DEBUG] train episode 171: reward = 26.00, steps = 26\n",
      "11:00:53 [DEBUG] train episode 172: reward = 46.00, steps = 46\n",
      "11:00:58 [DEBUG] train episode 173: reward = 45.00, steps = 45\n",
      "11:01:08 [DEBUG] train episode 174: reward = 112.00, steps = 112\n",
      "11:01:20 [DEBUG] train episode 175: reward = 119.00, steps = 119\n",
      "11:01:28 [DEBUG] train episode 176: reward = 84.00, steps = 84\n",
      "11:01:36 [DEBUG] train episode 177: reward = 82.00, steps = 82\n",
      "11:01:41 [DEBUG] train episode 178: reward = 55.00, steps = 55\n",
      "11:01:47 [DEBUG] train episode 179: reward = 63.00, steps = 63\n",
      "11:01:51 [DEBUG] train episode 180: reward = 34.00, steps = 34\n",
      "11:01:55 [DEBUG] train episode 181: reward = 47.00, steps = 47\n",
      "11:01:58 [DEBUG] train episode 182: reward = 28.00, steps = 28\n",
      "11:02:05 [DEBUG] train episode 183: reward = 70.00, steps = 70\n",
      "11:02:09 [DEBUG] train episode 184: reward = 31.00, steps = 31\n",
      "11:02:14 [DEBUG] train episode 185: reward = 53.00, steps = 53\n",
      "11:02:19 [DEBUG] train episode 186: reward = 49.00, steps = 49\n",
      "11:02:33 [DEBUG] train episode 187: reward = 152.00, steps = 152\n",
      "11:02:39 [DEBUG] train episode 188: reward = 49.00, steps = 49\n",
      "11:02:44 [DEBUG] train episode 189: reward = 53.00, steps = 53\n",
      "11:02:54 [DEBUG] train episode 190: reward = 101.00, steps = 101\n",
      "11:03:01 [DEBUG] train episode 191: reward = 75.00, steps = 75\n",
      "11:03:12 [DEBUG] train episode 192: reward = 111.00, steps = 111\n",
      "11:03:17 [DEBUG] train episode 193: reward = 46.00, steps = 46\n",
      "11:03:24 [DEBUG] train episode 194: reward = 71.00, steps = 71\n",
      "11:03:30 [DEBUG] train episode 195: reward = 58.00, steps = 58\n",
      "11:03:32 [DEBUG] train episode 196: reward = 20.00, steps = 20\n",
      "11:03:38 [DEBUG] train episode 197: reward = 63.00, steps = 63\n",
      "11:03:50 [DEBUG] train episode 198: reward = 117.00, steps = 117\n",
      "11:03:56 [DEBUG] train episode 199: reward = 69.00, steps = 69\n",
      "11:04:05 [DEBUG] train episode 200: reward = 78.00, steps = 78\n",
      "11:04:07 [DEBUG] train episode 201: reward = 28.00, steps = 28\n",
      "11:04:17 [DEBUG] train episode 202: reward = 93.00, steps = 93\n",
      "11:04:30 [DEBUG] train episode 203: reward = 134.00, steps = 134\n",
      "11:04:33 [DEBUG] train episode 204: reward = 29.00, steps = 29\n",
      "11:04:47 [DEBUG] train episode 205: reward = 143.00, steps = 143\n",
      "11:04:51 [DEBUG] train episode 206: reward = 41.00, steps = 41\n",
      "11:04:58 [DEBUG] train episode 207: reward = 71.00, steps = 71\n",
      "11:05:02 [DEBUG] train episode 208: reward = 38.00, steps = 38\n",
      "11:05:06 [DEBUG] train episode 209: reward = 40.00, steps = 40\n",
      "11:05:16 [DEBUG] train episode 210: reward = 104.00, steps = 104\n",
      "11:05:20 [DEBUG] train episode 211: reward = 37.00, steps = 37\n",
      "11:05:26 [DEBUG] train episode 212: reward = 59.00, steps = 59\n",
      "11:05:29 [DEBUG] train episode 213: reward = 25.00, steps = 25\n",
      "11:05:32 [DEBUG] train episode 214: reward = 32.00, steps = 32\n",
      "11:05:49 [DEBUG] train episode 215: reward = 177.00, steps = 177\n",
      "11:05:57 [DEBUG] train episode 216: reward = 77.00, steps = 77\n",
      "11:06:01 [DEBUG] train episode 217: reward = 29.00, steps = 29\n",
      "11:06:03 [DEBUG] train episode 218: reward = 21.00, steps = 21\n",
      "11:06:16 [DEBUG] train episode 219: reward = 140.00, steps = 140\n",
      "11:06:20 [DEBUG] train episode 220: reward = 32.00, steps = 32\n",
      "11:06:23 [DEBUG] train episode 221: reward = 29.00, steps = 29\n",
      "11:06:29 [DEBUG] train episode 222: reward = 58.00, steps = 58\n",
      "11:06:36 [DEBUG] train episode 223: reward = 74.00, steps = 74\n",
      "11:06:44 [DEBUG] train episode 224: reward = 87.00, steps = 87\n",
      "11:06:52 [DEBUG] train episode 225: reward = 86.00, steps = 86\n",
      "11:07:09 [DEBUG] train episode 226: reward = 175.00, steps = 175\n",
      "11:07:27 [DEBUG] train episode 227: reward = 188.00, steps = 188\n",
      "11:07:36 [DEBUG] train episode 228: reward = 91.00, steps = 91\n",
      "11:07:43 [DEBUG] train episode 229: reward = 74.00, steps = 74\n",
      "11:07:54 [DEBUG] train episode 230: reward = 103.00, steps = 103\n",
      "11:08:03 [DEBUG] train episode 231: reward = 106.00, steps = 106\n",
      "11:08:22 [DEBUG] train episode 232: reward = 200.00, steps = 200\n",
      "11:08:29 [DEBUG] train episode 233: reward = 64.00, steps = 64\n",
      "11:08:37 [DEBUG] train episode 234: reward = 82.00, steps = 82\n",
      "11:08:43 [DEBUG] train episode 235: reward = 60.00, steps = 60\n",
      "11:08:49 [DEBUG] train episode 236: reward = 68.00, steps = 68\n",
      "11:09:06 [DEBUG] train episode 237: reward = 179.00, steps = 179\n",
      "11:09:13 [DEBUG] train episode 238: reward = 67.00, steps = 67\n",
      "11:09:16 [DEBUG] train episode 239: reward = 28.00, steps = 28\n",
      "11:09:22 [DEBUG] train episode 240: reward = 62.00, steps = 62\n",
      "11:09:25 [DEBUG] train episode 241: reward = 24.00, steps = 24\n",
      "11:09:45 [DEBUG] train episode 242: reward = 200.00, steps = 200\n",
      "11:10:06 [DEBUG] train episode 243: reward = 200.00, steps = 200\n",
      "11:10:18 [DEBUG] train episode 244: reward = 107.00, steps = 107\n",
      "11:10:28 [DEBUG] train episode 245: reward = 90.00, steps = 90\n",
      "11:10:37 [DEBUG] train episode 246: reward = 89.00, steps = 89\n",
      "11:10:46 [DEBUG] train episode 247: reward = 98.00, steps = 98\n",
      "11:11:03 [DEBUG] train episode 248: reward = 168.00, steps = 168\n",
      "11:11:13 [DEBUG] train episode 249: reward = 108.00, steps = 108\n",
      "11:11:27 [DEBUG] train episode 250: reward = 134.00, steps = 134\n",
      "11:11:38 [DEBUG] train episode 251: reward = 103.00, steps = 103\n",
      "11:11:45 [DEBUG] train episode 252: reward = 59.00, steps = 59\n",
      "11:11:51 [DEBUG] train episode 253: reward = 60.00, steps = 60\n",
      "11:12:02 [DEBUG] train episode 254: reward = 110.00, steps = 110\n",
      "11:12:20 [DEBUG] train episode 255: reward = 162.00, steps = 162\n",
      "11:12:37 [DEBUG] train episode 256: reward = 166.00, steps = 166\n",
      "11:12:42 [DEBUG] train episode 257: reward = 35.00, steps = 35\n",
      "11:12:55 [DEBUG] train episode 258: reward = 118.00, steps = 118\n",
      "11:12:59 [DEBUG] train episode 259: reward = 34.00, steps = 34\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11:13:05 [DEBUG] train episode 260: reward = 60.00, steps = 60\n",
      "11:13:15 [DEBUG] train episode 261: reward = 110.00, steps = 110\n",
      "11:13:24 [DEBUG] train episode 262: reward = 88.00, steps = 88\n",
      "11:13:32 [DEBUG] train episode 263: reward = 85.00, steps = 85\n",
      "11:13:38 [DEBUG] train episode 264: reward = 68.00, steps = 68\n",
      "11:13:47 [DEBUG] train episode 265: reward = 94.00, steps = 94\n",
      "11:13:50 [DEBUG] train episode 266: reward = 16.00, steps = 16\n",
      "11:13:53 [DEBUG] train episode 267: reward = 32.00, steps = 32\n",
      "11:13:57 [DEBUG] train episode 268: reward = 39.00, steps = 39\n",
      "11:14:15 [DEBUG] train episode 269: reward = 196.00, steps = 196\n",
      "11:14:18 [DEBUG] train episode 270: reward = 22.00, steps = 22\n",
      "11:14:23 [DEBUG] train episode 271: reward = 44.00, steps = 44\n",
      "11:14:41 [DEBUG] train episode 272: reward = 200.00, steps = 200\n",
      "11:14:47 [DEBUG] train episode 273: reward = 59.00, steps = 59\n",
      "11:14:57 [DEBUG] train episode 274: reward = 116.00, steps = 116\n",
      "11:15:09 [DEBUG] train episode 275: reward = 119.00, steps = 119\n",
      "11:15:20 [DEBUG] train episode 276: reward = 126.00, steps = 126\n",
      "11:15:36 [DEBUG] train episode 277: reward = 174.00, steps = 174\n",
      "11:15:55 [DEBUG] train episode 278: reward = 200.00, steps = 200\n",
      "11:16:11 [DEBUG] train episode 279: reward = 175.00, steps = 175\n",
      "11:16:19 [DEBUG] train episode 280: reward = 78.00, steps = 78\n",
      "11:16:32 [DEBUG] train episode 281: reward = 136.00, steps = 136\n",
      "11:16:45 [DEBUG] train episode 282: reward = 139.00, steps = 139\n",
      "11:16:47 [DEBUG] train episode 283: reward = 15.00, steps = 15\n",
      "11:16:56 [DEBUG] train episode 284: reward = 88.00, steps = 88\n",
      "11:17:01 [DEBUG] train episode 285: reward = 52.00, steps = 52\n",
      "11:17:04 [DEBUG] train episode 286: reward = 22.00, steps = 22\n",
      "11:17:25 [DEBUG] train episode 287: reward = 200.00, steps = 200\n",
      "11:17:28 [DEBUG] train episode 288: reward = 27.00, steps = 27\n",
      "11:17:44 [DEBUG] train episode 289: reward = 169.00, steps = 169\n",
      "11:18:02 [DEBUG] train episode 290: reward = 200.00, steps = 200\n",
      "11:18:12 [DEBUG] train episode 291: reward = 101.00, steps = 101\n",
      "11:18:30 [DEBUG] train episode 292: reward = 200.00, steps = 200\n",
      "11:18:34 [DEBUG] train episode 293: reward = 37.00, steps = 37\n",
      "11:18:47 [DEBUG] train episode 294: reward = 146.00, steps = 146\n",
      "11:18:52 [DEBUG] train episode 295: reward = 45.00, steps = 45\n",
      "11:19:10 [DEBUG] train episode 296: reward = 200.00, steps = 200\n",
      "11:19:20 [DEBUG] train episode 297: reward = 111.00, steps = 111\n",
      "11:19:32 [DEBUG] train episode 298: reward = 125.00, steps = 125\n",
      "11:19:46 [DEBUG] train episode 299: reward = 145.00, steps = 145\n",
      "11:19:59 [DEBUG] train episode 300: reward = 139.00, steps = 139\n",
      "11:20:17 [DEBUG] train episode 301: reward = 182.00, steps = 182\n",
      "11:20:39 [DEBUG] train episode 302: reward = 200.00, steps = 200\n",
      "11:21:00 [DEBUG] train episode 303: reward = 199.00, steps = 199\n",
      "11:21:10 [DEBUG] train episode 304: reward = 112.00, steps = 112\n",
      "11:21:15 [DEBUG] train episode 305: reward = 45.00, steps = 45\n",
      "11:21:27 [DEBUG] train episode 306: reward = 128.00, steps = 128\n",
      "11:21:30 [DEBUG] train episode 307: reward = 30.00, steps = 30\n",
      "11:21:47 [DEBUG] train episode 308: reward = 183.00, steps = 183\n",
      "11:21:50 [DEBUG] train episode 309: reward = 27.00, steps = 27\n",
      "11:22:03 [DEBUG] train episode 310: reward = 147.00, steps = 147\n",
      "11:22:22 [DEBUG] train episode 311: reward = 200.00, steps = 200\n",
      "11:22:40 [DEBUG] train episode 312: reward = 200.00, steps = 200\n",
      "11:22:58 [DEBUG] train episode 313: reward = 200.00, steps = 200\n",
      "11:23:15 [DEBUG] train episode 314: reward = 175.00, steps = 175\n",
      "11:23:23 [DEBUG] train episode 315: reward = 90.00, steps = 90\n",
      "11:23:31 [DEBUG] train episode 316: reward = 82.00, steps = 82\n",
      "11:23:44 [DEBUG] train episode 317: reward = 141.00, steps = 141\n",
      "11:24:01 [DEBUG] train episode 318: reward = 187.00, steps = 187\n",
      "11:24:06 [DEBUG] train episode 319: reward = 49.00, steps = 49\n",
      "11:24:24 [DEBUG] train episode 320: reward = 200.00, steps = 200\n",
      "11:24:41 [DEBUG] train episode 321: reward = 178.00, steps = 178\n",
      "11:24:52 [DEBUG] train episode 322: reward = 125.00, steps = 125\n",
      "11:25:09 [DEBUG] train episode 323: reward = 177.00, steps = 177\n",
      "11:25:13 [DEBUG] train episode 324: reward = 37.00, steps = 37\n",
      "11:25:16 [DEBUG] train episode 325: reward = 28.00, steps = 28\n",
      "11:25:25 [DEBUG] train episode 326: reward = 96.00, steps = 96\n",
      "11:25:27 [DEBUG] train episode 327: reward = 16.00, steps = 16\n",
      "11:25:32 [DEBUG] train episode 328: reward = 56.00, steps = 56\n",
      "11:25:35 [DEBUG] train episode 329: reward = 24.00, steps = 24\n",
      "11:25:48 [DEBUG] train episode 330: reward = 143.00, steps = 143\n",
      "11:25:59 [DEBUG] train episode 331: reward = 121.00, steps = 121\n",
      "11:26:18 [DEBUG] train episode 332: reward = 200.00, steps = 200\n",
      "11:26:35 [DEBUG] train episode 333: reward = 185.00, steps = 185\n",
      "11:26:48 [DEBUG] train episode 334: reward = 145.00, steps = 145\n",
      "11:26:53 [DEBUG] train episode 335: reward = 47.00, steps = 47\n",
      "11:27:07 [DEBUG] train episode 336: reward = 148.00, steps = 148\n",
      "11:27:25 [DEBUG] train episode 337: reward = 200.00, steps = 200\n",
      "11:27:36 [DEBUG] train episode 338: reward = 123.00, steps = 123\n",
      "11:27:42 [DEBUG] train episode 339: reward = 55.00, steps = 55\n",
      "11:27:53 [DEBUG] train episode 340: reward = 126.00, steps = 126\n",
      "11:28:10 [DEBUG] train episode 341: reward = 179.00, steps = 179\n",
      "11:28:28 [DEBUG] train episode 342: reward = 197.00, steps = 197\n",
      "11:28:31 [DEBUG] train episode 343: reward = 29.00, steps = 29\n",
      "11:28:45 [DEBUG] train episode 344: reward = 152.00, steps = 152\n",
      "11:28:58 [DEBUG] train episode 345: reward = 144.00, steps = 144\n",
      "11:29:03 [DEBUG] train episode 346: reward = 45.00, steps = 45\n",
      "11:29:13 [DEBUG] train episode 347: reward = 116.00, steps = 116\n",
      "11:29:27 [DEBUG] train episode 348: reward = 141.00, steps = 141\n",
      "11:29:40 [DEBUG] train episode 349: reward = 149.00, steps = 149\n",
      "11:29:59 [DEBUG] train episode 350: reward = 200.00, steps = 200\n",
      "11:30:17 [DEBUG] train episode 351: reward = 200.00, steps = 200\n",
      "11:30:28 [DEBUG] train episode 352: reward = 125.00, steps = 125\n",
      "11:30:47 [DEBUG] train episode 353: reward = 200.00, steps = 200\n",
      "11:30:56 [DEBUG] train episode 354: reward = 97.00, steps = 97\n",
      "11:31:07 [DEBUG] train episode 355: reward = 118.00, steps = 118\n",
      "11:31:25 [DEBUG] train episode 356: reward = 200.00, steps = 200\n",
      "11:31:37 [DEBUG] train episode 357: reward = 125.00, steps = 125\n",
      "11:31:44 [DEBUG] train episode 358: reward = 75.00, steps = 75\n",
      "11:31:56 [DEBUG] train episode 359: reward = 126.00, steps = 126\n",
      "11:32:07 [DEBUG] train episode 360: reward = 127.00, steps = 127\n",
      "11:32:11 [DEBUG] train episode 361: reward = 35.00, steps = 35\n",
      "11:32:16 [DEBUG] train episode 362: reward = 59.00, steps = 59\n",
      "11:32:30 [DEBUG] train episode 363: reward = 145.00, steps = 145\n",
      "11:32:34 [DEBUG] train episode 364: reward = 44.00, steps = 44\n",
      "11:32:52 [DEBUG] train episode 365: reward = 194.00, steps = 194\n",
      "11:33:12 [DEBUG] train episode 366: reward = 200.00, steps = 200\n",
      "11:33:28 [DEBUG] train episode 367: reward = 174.00, steps = 174\n",
      "11:33:31 [DEBUG] train episode 368: reward = 28.00, steps = 28\n",
      "11:33:51 [DEBUG] train episode 369: reward = 199.00, steps = 199\n",
      "11:34:10 [DEBUG] train episode 370: reward = 194.00, steps = 194\n",
      "11:34:14 [DEBUG] train episode 371: reward = 37.00, steps = 37\n",
      "11:34:28 [DEBUG] train episode 372: reward = 151.00, steps = 151\n",
      "11:34:46 [DEBUG] train episode 373: reward = 200.00, steps = 200\n",
      "11:35:04 [DEBUG] train episode 374: reward = 200.00, steps = 200\n",
      "11:35:20 [DEBUG] train episode 375: reward = 165.00, steps = 165\n",
      "11:35:38 [DEBUG] train episode 376: reward = 200.00, steps = 200\n",
      "11:35:47 [DEBUG] train episode 377: reward = 89.00, steps = 89\n",
      "11:36:05 [DEBUG] train episode 378: reward = 200.00, steps = 200\n",
      "11:36:23 [DEBUG] train episode 379: reward = 200.00, steps = 200\n",
      "11:36:42 [DEBUG] train episode 380: reward = 200.00, steps = 200\n",
      "11:37:00 [DEBUG] train episode 381: reward = 200.00, steps = 200\n",
      "11:37:18 [DEBUG] train episode 382: reward = 200.00, steps = 200\n",
      "11:37:30 [DEBUG] train episode 383: reward = 125.00, steps = 125\n",
      "11:37:48 [DEBUG] train episode 384: reward = 200.00, steps = 200\n",
      "11:38:07 [DEBUG] train episode 385: reward = 200.00, steps = 200\n",
      "11:38:25 [DEBUG] train episode 386: reward = 195.00, steps = 195\n",
      "11:38:43 [DEBUG] train episode 387: reward = 200.00, steps = 200\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11:38:57 [DEBUG] train episode 388: reward = 154.00, steps = 154\n",
      "11:39:15 [DEBUG] train episode 389: reward = 192.00, steps = 192\n",
      "11:39:33 [DEBUG] train episode 390: reward = 200.00, steps = 200\n",
      "11:39:51 [DEBUG] train episode 391: reward = 200.00, steps = 200\n",
      "11:40:08 [DEBUG] train episode 392: reward = 183.00, steps = 183\n",
      "11:40:26 [DEBUG] train episode 393: reward = 197.00, steps = 197\n",
      "11:40:45 [DEBUG] train episode 394: reward = 200.00, steps = 200\n",
      "11:41:03 [DEBUG] train episode 395: reward = 200.00, steps = 200\n",
      "11:41:22 [DEBUG] train episode 396: reward = 200.00, steps = 200\n",
      "11:41:40 [DEBUG] train episode 397: reward = 200.00, steps = 200\n",
      "11:41:52 [DEBUG] train episode 398: reward = 130.00, steps = 130\n",
      "11:42:10 [DEBUG] train episode 399: reward = 200.00, steps = 200\n",
      "11:42:29 [DEBUG] train episode 400: reward = 200.00, steps = 200\n",
      "11:42:35 [DEBUG] train episode 401: reward = 74.00, steps = 74\n",
      "11:42:51 [DEBUG] train episode 402: reward = 163.00, steps = 163\n",
      "11:43:03 [DEBUG] train episode 403: reward = 135.00, steps = 135\n",
      "11:43:21 [DEBUG] train episode 404: reward = 200.00, steps = 200\n",
      "11:43:35 [DEBUG] train episode 405: reward = 154.00, steps = 154\n",
      "11:43:47 [DEBUG] train episode 406: reward = 124.00, steps = 124\n",
      "11:43:57 [DEBUG] train episode 407: reward = 114.00, steps = 114\n",
      "11:44:07 [DEBUG] train episode 408: reward = 111.00, steps = 111\n",
      "11:44:18 [DEBUG] train episode 409: reward = 117.00, steps = 117\n",
      "11:44:31 [DEBUG] train episode 410: reward = 137.00, steps = 137\n",
      "11:44:41 [DEBUG] train episode 411: reward = 105.00, steps = 105\n",
      "11:44:54 [DEBUG] train episode 412: reward = 137.00, steps = 137\n",
      "11:45:08 [DEBUG] train episode 413: reward = 151.00, steps = 151\n",
      "11:45:26 [DEBUG] train episode 414: reward = 200.00, steps = 200\n",
      "11:45:44 [DEBUG] train episode 415: reward = 200.00, steps = 200\n",
      "11:45:55 [DEBUG] train episode 416: reward = 118.00, steps = 118\n",
      "11:46:07 [DEBUG] train episode 417: reward = 126.00, steps = 126\n",
      "11:46:20 [DEBUG] train episode 418: reward = 136.00, steps = 136\n",
      "11:46:28 [DEBUG] train episode 419: reward = 86.00, steps = 86\n",
      "11:46:33 [DEBUG] train episode 420: reward = 56.00, steps = 56\n",
      "11:46:47 [DEBUG] train episode 421: reward = 150.00, steps = 150\n",
      "11:46:51 [DEBUG] train episode 422: reward = 34.00, steps = 34\n",
      "11:46:57 [DEBUG] train episode 423: reward = 66.00, steps = 66\n",
      "11:47:16 [DEBUG] train episode 424: reward = 200.00, steps = 200\n",
      "11:47:33 [DEBUG] train episode 425: reward = 196.00, steps = 196\n",
      "11:47:47 [DEBUG] train episode 426: reward = 145.00, steps = 145\n",
      "11:47:59 [DEBUG] train episode 427: reward = 131.00, steps = 131\n",
      "11:48:09 [DEBUG] train episode 428: reward = 108.00, steps = 108\n",
      "11:48:27 [DEBUG] train episode 429: reward = 188.00, steps = 188\n",
      "11:48:45 [DEBUG] train episode 430: reward = 200.00, steps = 200\n",
      "11:49:00 [DEBUG] train episode 431: reward = 163.00, steps = 163\n",
      "11:49:11 [DEBUG] train episode 432: reward = 117.00, steps = 117\n",
      "11:49:25 [DEBUG] train episode 433: reward = 151.00, steps = 151\n",
      "11:49:43 [DEBUG] train episode 434: reward = 200.00, steps = 200\n",
      "11:49:55 [DEBUG] train episode 435: reward = 123.00, steps = 123\n",
      "11:50:13 [DEBUG] train episode 436: reward = 199.00, steps = 199\n",
      "11:50:31 [DEBUG] train episode 437: reward = 200.00, steps = 200\n",
      "11:50:42 [DEBUG] train episode 438: reward = 114.00, steps = 114\n",
      "11:50:50 [DEBUG] train episode 439: reward = 90.00, steps = 90\n",
      "11:51:09 [DEBUG] train episode 440: reward = 198.00, steps = 198\n",
      "11:51:19 [DEBUG] train episode 441: reward = 114.00, steps = 114\n",
      "11:51:38 [DEBUG] train episode 442: reward = 200.00, steps = 200\n",
      "11:51:56 [DEBUG] train episode 443: reward = 200.00, steps = 200\n",
      "11:52:14 [DEBUG] train episode 444: reward = 200.00, steps = 200\n",
      "11:52:33 [DEBUG] train episode 445: reward = 200.00, steps = 200\n",
      "11:52:51 [DEBUG] train episode 446: reward = 200.00, steps = 200\n",
      "11:53:04 [DEBUG] train episode 447: reward = 137.00, steps = 137\n",
      "11:53:22 [DEBUG] train episode 448: reward = 200.00, steps = 200\n",
      "11:53:40 [DEBUG] train episode 449: reward = 200.00, steps = 200\n",
      "11:53:52 [DEBUG] train episode 450: reward = 130.00, steps = 130\n",
      "11:53:58 [DEBUG] train episode 451: reward = 57.00, steps = 57\n",
      "11:54:16 [DEBUG] train episode 452: reward = 200.00, steps = 200\n",
      "11:54:28 [DEBUG] train episode 453: reward = 135.00, steps = 135\n",
      "11:54:47 [DEBUG] train episode 454: reward = 200.00, steps = 200\n",
      "11:54:58 [DEBUG] train episode 455: reward = 124.00, steps = 124\n",
      "11:55:17 [DEBUG] train episode 456: reward = 200.00, steps = 200\n",
      "11:55:27 [DEBUG] train episode 457: reward = 110.00, steps = 110\n",
      "11:55:45 [DEBUG] train episode 458: reward = 200.00, steps = 200\n",
      "11:56:03 [DEBUG] train episode 459: reward = 197.00, steps = 197\n",
      "11:56:14 [DEBUG] train episode 460: reward = 110.00, steps = 110\n",
      "11:56:31 [DEBUG] train episode 461: reward = 189.00, steps = 189\n",
      "11:56:47 [DEBUG] train episode 462: reward = 170.00, steps = 170\n",
      "11:56:58 [DEBUG] train episode 463: reward = 118.00, steps = 118\n",
      "11:57:16 [DEBUG] train episode 464: reward = 196.00, steps = 196\n",
      "11:57:28 [DEBUG] train episode 465: reward = 127.00, steps = 127\n",
      "11:57:30 [DEBUG] train episode 466: reward = 22.00, steps = 22\n",
      "11:57:48 [DEBUG] train episode 467: reward = 200.00, steps = 200\n",
      "11:58:07 [DEBUG] train episode 468: reward = 200.00, steps = 200\n",
      "11:58:25 [DEBUG] train episode 469: reward = 200.00, steps = 200\n",
      "11:58:33 [DEBUG] train episode 470: reward = 86.00, steps = 86\n",
      "11:58:48 [DEBUG] train episode 471: reward = 167.00, steps = 167\n",
      "11:58:54 [DEBUG] train episode 472: reward = 59.00, steps = 59\n",
      "11:59:10 [DEBUG] train episode 473: reward = 176.00, steps = 176\n",
      "11:59:29 [DEBUG] train episode 474: reward = 200.00, steps = 200\n",
      "11:59:47 [DEBUG] train episode 475: reward = 200.00, steps = 200\n",
      "12:00:05 [DEBUG] train episode 476: reward = 200.00, steps = 200\n",
      "12:00:24 [DEBUG] train episode 477: reward = 200.00, steps = 200\n",
      "12:00:38 [DEBUG] train episode 478: reward = 153.00, steps = 153\n",
      "12:00:56 [DEBUG] train episode 479: reward = 200.00, steps = 200\n",
      "12:01:15 [DEBUG] train episode 480: reward = 200.00, steps = 200\n",
      "12:01:28 [DEBUG] train episode 481: reward = 138.00, steps = 138\n",
      "12:01:42 [DEBUG] train episode 482: reward = 155.00, steps = 155\n",
      "12:02:00 [DEBUG] train episode 483: reward = 200.00, steps = 200\n",
      "12:02:13 [DEBUG] train episode 484: reward = 128.00, steps = 128\n",
      "12:02:31 [DEBUG] train episode 485: reward = 200.00, steps = 200\n",
      "12:02:49 [DEBUG] train episode 486: reward = 200.00, steps = 200\n",
      "12:03:07 [DEBUG] train episode 487: reward = 200.00, steps = 200\n",
      "12:03:22 [DEBUG] train episode 488: reward = 153.00, steps = 153\n",
      "12:03:28 [DEBUG] train episode 489: reward = 64.00, steps = 64\n",
      "12:03:46 [DEBUG] train episode 490: reward = 200.00, steps = 200\n",
      "12:04:05 [DEBUG] train episode 491: reward = 200.00, steps = 200\n",
      "12:04:10 [DEBUG] train episode 492: reward = 62.00, steps = 62\n",
      "12:04:19 [DEBUG] train episode 493: reward = 87.00, steps = 87\n",
      "12:04:32 [DEBUG] train episode 494: reward = 144.00, steps = 144\n",
      "12:04:50 [DEBUG] train episode 495: reward = 200.00, steps = 200\n",
      "12:05:06 [DEBUG] train episode 496: reward = 168.00, steps = 168\n",
      "12:05:24 [DEBUG] train episode 497: reward = 200.00, steps = 200\n",
      "12:05:36 [DEBUG] train episode 498: reward = 121.00, steps = 121\n",
      "12:05:46 [DEBUG] train episode 499: reward = 112.00, steps = 112\n",
      "12:06:00 [DEBUG] train episode 500: reward = 151.00, steps = 151\n",
      "12:06:15 [DEBUG] train episode 501: reward = 155.00, steps = 155\n",
      "12:06:33 [DEBUG] train episode 502: reward = 200.00, steps = 200\n",
      "12:06:46 [DEBUG] train episode 503: reward = 141.00, steps = 141\n",
      "12:07:04 [DEBUG] train episode 504: reward = 200.00, steps = 200\n",
      "12:07:18 [DEBUG] train episode 505: reward = 150.00, steps = 150\n",
      "12:07:31 [DEBUG] train episode 506: reward = 139.00, steps = 139\n",
      "12:07:49 [DEBUG] train episode 507: reward = 200.00, steps = 200\n",
      "12:08:07 [DEBUG] train episode 508: reward = 200.00, steps = 200\n",
      "12:08:19 [DEBUG] train episode 509: reward = 126.00, steps = 126\n",
      "12:08:38 [DEBUG] train episode 510: reward = 200.00, steps = 200\n",
      "12:08:56 [DEBUG] train episode 511: reward = 200.00, steps = 200\n",
      "12:09:14 [DEBUG] train episode 512: reward = 200.00, steps = 200\n",
      "12:09:27 [DEBUG] train episode 513: reward = 136.00, steps = 136\n",
      "12:09:32 [DEBUG] train episode 514: reward = 57.00, steps = 57\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12:09:43 [DEBUG] train episode 515: reward = 109.00, steps = 109\n",
      "12:10:00 [DEBUG] train episode 516: reward = 196.00, steps = 196\n",
      "12:10:16 [DEBUG] train episode 517: reward = 166.00, steps = 166\n",
      "12:10:34 [DEBUG] train episode 518: reward = 200.00, steps = 200\n",
      "12:10:45 [DEBUG] train episode 519: reward = 117.00, steps = 117\n",
      "12:11:00 [DEBUG] train episode 520: reward = 163.00, steps = 163\n",
      "12:11:15 [DEBUG] train episode 521: reward = 164.00, steps = 164\n",
      "12:11:29 [DEBUG] train episode 522: reward = 143.00, steps = 143\n",
      "12:11:38 [DEBUG] train episode 523: reward = 105.00, steps = 105\n",
      "12:11:56 [DEBUG] train episode 524: reward = 196.00, steps = 196\n",
      "12:12:10 [DEBUG] train episode 525: reward = 154.00, steps = 154\n",
      "12:12:19 [DEBUG] train episode 526: reward = 99.00, steps = 99\n",
      "12:12:31 [DEBUG] train episode 527: reward = 124.00, steps = 124\n",
      "12:12:42 [DEBUG] train episode 528: reward = 120.00, steps = 120\n",
      "12:12:55 [DEBUG] train episode 529: reward = 140.00, steps = 140\n",
      "12:13:11 [DEBUG] train episode 530: reward = 180.00, steps = 180\n",
      "12:13:28 [DEBUG] train episode 531: reward = 180.00, steps = 180\n",
      "12:13:39 [DEBUG] train episode 532: reward = 115.00, steps = 115\n",
      "12:13:49 [DEBUG] train episode 533: reward = 106.00, steps = 106\n",
      "12:14:04 [DEBUG] train episode 534: reward = 165.00, steps = 165\n",
      "12:14:19 [DEBUG] train episode 535: reward = 159.00, steps = 159\n",
      "12:14:24 [DEBUG] train episode 536: reward = 56.00, steps = 56\n",
      "12:14:34 [DEBUG] train episode 537: reward = 108.00, steps = 108\n",
      "12:14:48 [DEBUG] train episode 538: reward = 152.00, steps = 152\n",
      "12:14:59 [DEBUG] train episode 539: reward = 112.00, steps = 112\n",
      "12:15:15 [DEBUG] train episode 540: reward = 175.00, steps = 175\n",
      "12:15:30 [DEBUG] train episode 541: reward = 166.00, steps = 166\n",
      "12:15:44 [DEBUG] train episode 542: reward = 149.00, steps = 149\n",
      "12:15:56 [DEBUG] train episode 543: reward = 126.00, steps = 126\n",
      "12:16:09 [DEBUG] train episode 544: reward = 145.00, steps = 145\n",
      "12:16:27 [DEBUG] train episode 545: reward = 200.00, steps = 200\n",
      "12:16:42 [DEBUG] train episode 546: reward = 157.00, steps = 157\n",
      "12:17:00 [DEBUG] train episode 547: reward = 200.00, steps = 200\n",
      "12:17:12 [DEBUG] train episode 548: reward = 125.00, steps = 125\n",
      "12:17:21 [DEBUG] train episode 549: reward = 98.00, steps = 98\n",
      "12:17:33 [DEBUG] train episode 550: reward = 131.00, steps = 131\n",
      "12:17:50 [DEBUG] train episode 551: reward = 189.00, steps = 189\n",
      "12:18:09 [DEBUG] train episode 552: reward = 200.00, steps = 200\n",
      "12:18:27 [DEBUG] train episode 553: reward = 200.00, steps = 200\n",
      "12:18:42 [DEBUG] train episode 554: reward = 161.00, steps = 161\n",
      "12:19:00 [DEBUG] train episode 555: reward = 200.00, steps = 200\n",
      "12:19:19 [DEBUG] train episode 556: reward = 200.00, steps = 200\n",
      "12:19:34 [DEBUG] train episode 557: reward = 163.00, steps = 163\n",
      "12:19:45 [DEBUG] train episode 558: reward = 119.00, steps = 119\n",
      "12:20:03 [DEBUG] train episode 559: reward = 200.00, steps = 200\n",
      "12:20:21 [DEBUG] train episode 560: reward = 200.00, steps = 200\n",
      "12:20:36 [DEBUG] train episode 561: reward = 157.00, steps = 157\n",
      "12:20:53 [DEBUG] train episode 562: reward = 187.00, steps = 187\n",
      "12:21:10 [DEBUG] train episode 563: reward = 183.00, steps = 183\n",
      "12:21:28 [DEBUG] train episode 564: reward = 200.00, steps = 200\n",
      "12:21:43 [DEBUG] train episode 565: reward = 157.00, steps = 157\n",
      "12:21:57 [DEBUG] train episode 566: reward = 158.00, steps = 158\n",
      "12:22:16 [DEBUG] train episode 567: reward = 200.00, steps = 200\n",
      "12:22:33 [DEBUG] train episode 568: reward = 183.00, steps = 183\n",
      "12:22:51 [DEBUG] train episode 569: reward = 199.00, steps = 199\n",
      "12:23:09 [DEBUG] train episode 570: reward = 200.00, steps = 200\n",
      "12:23:27 [DEBUG] train episode 571: reward = 200.00, steps = 200\n",
      "12:23:44 [DEBUG] train episode 572: reward = 180.00, steps = 180\n",
      "12:23:57 [DEBUG] train episode 573: reward = 142.00, steps = 142\n",
      "12:24:15 [DEBUG] train episode 574: reward = 200.00, steps = 200\n",
      "12:24:33 [DEBUG] train episode 575: reward = 200.00, steps = 200\n",
      "12:24:50 [DEBUG] train episode 576: reward = 183.00, steps = 183\n",
      "12:25:08 [DEBUG] train episode 577: reward = 200.00, steps = 200\n",
      "12:25:20 [DEBUG] train episode 578: reward = 126.00, steps = 126\n",
      "12:25:31 [DEBUG] train episode 579: reward = 119.00, steps = 119\n",
      "12:25:44 [DEBUG] train episode 580: reward = 137.00, steps = 137\n",
      "12:26:02 [DEBUG] train episode 581: reward = 200.00, steps = 200\n",
      "12:26:20 [DEBUG] train episode 582: reward = 200.00, steps = 200\n",
      "12:26:34 [DEBUG] train episode 583: reward = 154.00, steps = 154\n",
      "12:26:53 [DEBUG] train episode 584: reward = 200.00, steps = 200\n",
      "12:27:11 [DEBUG] train episode 585: reward = 200.00, steps = 200\n",
      "12:27:29 [DEBUG] train episode 586: reward = 200.00, steps = 200\n",
      "12:27:46 [DEBUG] train episode 587: reward = 185.00, steps = 185\n",
      "12:28:04 [DEBUG] train episode 588: reward = 200.00, steps = 200\n",
      "12:28:22 [DEBUG] train episode 589: reward = 200.00, steps = 200\n",
      "12:28:31 [DEBUG] train episode 590: reward = 92.00, steps = 92\n",
      "12:28:45 [DEBUG] train episode 591: reward = 149.00, steps = 149\n",
      "12:28:55 [DEBUG] train episode 592: reward = 111.00, steps = 111\n",
      "12:29:13 [DEBUG] train episode 593: reward = 200.00, steps = 200\n",
      "12:29:31 [DEBUG] train episode 594: reward = 200.00, steps = 200\n",
      "12:29:45 [DEBUG] train episode 595: reward = 151.00, steps = 151\n",
      "12:29:59 [DEBUG] train episode 596: reward = 159.00, steps = 159\n",
      "12:30:13 [DEBUG] train episode 597: reward = 147.00, steps = 147\n",
      "12:30:31 [DEBUG] train episode 598: reward = 200.00, steps = 200\n",
      "12:30:49 [DEBUG] train episode 599: reward = 200.00, steps = 200\n",
      "12:31:05 [DEBUG] train episode 600: reward = 171.00, steps = 171\n",
      "12:31:08 [DEBUG] train episode 601: reward = 27.00, steps = 27\n",
      "12:31:26 [DEBUG] train episode 602: reward = 200.00, steps = 200\n",
      "12:31:33 [DEBUG] train episode 603: reward = 73.00, steps = 73\n",
      "12:31:48 [DEBUG] train episode 604: reward = 166.00, steps = 166\n",
      "12:32:02 [DEBUG] train episode 605: reward = 152.00, steps = 152\n",
      "12:32:21 [DEBUG] train episode 606: reward = 200.00, steps = 200\n",
      "12:32:39 [DEBUG] train episode 607: reward = 200.00, steps = 200\n",
      "12:32:57 [DEBUG] train episode 608: reward = 200.00, steps = 200\n",
      "12:33:15 [DEBUG] train episode 609: reward = 200.00, steps = 200\n",
      "12:33:26 [DEBUG] train episode 610: reward = 119.00, steps = 119\n",
      "12:33:45 [DEBUG] train episode 611: reward = 200.00, steps = 200\n",
      "12:33:58 [DEBUG] train episode 612: reward = 148.00, steps = 148\n",
      "12:34:15 [DEBUG] train episode 613: reward = 182.00, steps = 182\n",
      "12:34:33 [DEBUG] train episode 614: reward = 200.00, steps = 200\n",
      "12:34:52 [DEBUG] train episode 615: reward = 200.00, steps = 200\n",
      "12:35:10 [DEBUG] train episode 616: reward = 200.00, steps = 200\n",
      "12:35:24 [DEBUG] train episode 617: reward = 149.00, steps = 149\n",
      "12:35:37 [DEBUG] train episode 618: reward = 135.00, steps = 135\n",
      "12:35:48 [DEBUG] train episode 619: reward = 128.00, steps = 128\n",
      "12:36:01 [DEBUG] train episode 620: reward = 131.00, steps = 131\n",
      "12:36:19 [DEBUG] train episode 621: reward = 200.00, steps = 200\n",
      "12:36:37 [DEBUG] train episode 622: reward = 200.00, steps = 200\n",
      "12:36:57 [DEBUG] train episode 623: reward = 200.00, steps = 200\n",
      "12:37:16 [DEBUG] train episode 624: reward = 200.00, steps = 200\n",
      "12:37:28 [DEBUG] train episode 625: reward = 133.00, steps = 133\n",
      "12:37:46 [DEBUG] train episode 626: reward = 200.00, steps = 200\n",
      "12:38:00 [DEBUG] train episode 627: reward = 152.00, steps = 152\n",
      "12:38:18 [DEBUG] train episode 628: reward = 200.00, steps = 200\n",
      "12:38:29 [DEBUG] train episode 629: reward = 112.00, steps = 112\n",
      "12:38:45 [DEBUG] train episode 630: reward = 159.00, steps = 159\n",
      "12:38:58 [DEBUG] train episode 631: reward = 140.00, steps = 140\n",
      "12:39:16 [DEBUG] train episode 632: reward = 200.00, steps = 200\n",
      "12:39:29 [DEBUG] train episode 633: reward = 144.00, steps = 144\n",
      "12:39:42 [DEBUG] train episode 634: reward = 140.00, steps = 140\n",
      "12:40:01 [DEBUG] train episode 635: reward = 200.00, steps = 200\n",
      "12:40:13 [DEBUG] train episode 636: reward = 130.00, steps = 130\n",
      "12:40:32 [DEBUG] train episode 637: reward = 200.00, steps = 200\n",
      "12:40:47 [DEBUG] train episode 638: reward = 162.00, steps = 162\n",
      "12:41:05 [DEBUG] train episode 639: reward = 200.00, steps = 200\n",
      "12:41:23 [DEBUG] train episode 640: reward = 200.00, steps = 200\n",
      "12:41:38 [DEBUG] train episode 641: reward = 161.00, steps = 161\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12:41:49 [DEBUG] train episode 642: reward = 117.00, steps = 117\n",
      "12:41:57 [DEBUG] train episode 643: reward = 75.00, steps = 75\n",
      "12:42:16 [DEBUG] train episode 644: reward = 200.00, steps = 200\n",
      "12:42:36 [DEBUG] train episode 645: reward = 200.00, steps = 200\n",
      "12:42:54 [DEBUG] train episode 646: reward = 200.00, steps = 200\n",
      "12:43:13 [DEBUG] train episode 647: reward = 200.00, steps = 200\n",
      "12:43:31 [DEBUG] train episode 648: reward = 200.00, steps = 200\n",
      "12:43:47 [DEBUG] train episode 649: reward = 170.00, steps = 170\n",
      "12:44:01 [DEBUG] train episode 650: reward = 140.00, steps = 140\n",
      "12:44:20 [DEBUG] train episode 651: reward = 200.00, steps = 200\n",
      "12:44:30 [DEBUG] train episode 652: reward = 99.00, steps = 99\n",
      "12:44:44 [DEBUG] train episode 653: reward = 159.00, steps = 159\n",
      "12:45:02 [DEBUG] train episode 654: reward = 195.00, steps = 195\n",
      "12:45:20 [DEBUG] train episode 655: reward = 200.00, steps = 200\n",
      "12:45:31 [DEBUG] train episode 656: reward = 120.00, steps = 120\n",
      "12:45:49 [DEBUG] train episode 657: reward = 200.00, steps = 200\n",
      "12:46:07 [DEBUG] train episode 658: reward = 200.00, steps = 200\n",
      "12:46:21 [DEBUG] train episode 659: reward = 147.00, steps = 147\n",
      "12:46:34 [DEBUG] train episode 660: reward = 145.00, steps = 145\n",
      "12:46:52 [DEBUG] train episode 661: reward = 200.00, steps = 200\n",
      "12:47:11 [DEBUG] train episode 662: reward = 200.00, steps = 200\n",
      "12:47:29 [DEBUG] train episode 663: reward = 200.00, steps = 200\n",
      "12:47:47 [DEBUG] train episode 664: reward = 200.00, steps = 200\n",
      "12:48:03 [DEBUG] train episode 665: reward = 176.00, steps = 176\n",
      "12:48:17 [DEBUG] train episode 666: reward = 148.00, steps = 148\n",
      "12:48:34 [DEBUG] train episode 667: reward = 190.00, steps = 190\n",
      "12:48:45 [DEBUG] train episode 668: reward = 106.00, steps = 106\n",
      "12:49:03 [DEBUG] train episode 669: reward = 200.00, steps = 200\n",
      "12:49:21 [DEBUG] train episode 670: reward = 200.00, steps = 200\n",
      "12:49:39 [DEBUG] train episode 671: reward = 200.00, steps = 200\n",
      "12:49:56 [DEBUG] train episode 672: reward = 183.00, steps = 183\n",
      "12:50:15 [DEBUG] train episode 673: reward = 200.00, steps = 200\n",
      "12:50:29 [DEBUG] train episode 674: reward = 162.00, steps = 162\n",
      "12:50:47 [DEBUG] train episode 675: reward = 200.00, steps = 200\n",
      "12:51:04 [DEBUG] train episode 676: reward = 188.00, steps = 188\n",
      "12:51:19 [DEBUG] train episode 677: reward = 158.00, steps = 158\n",
      "12:51:37 [DEBUG] train episode 678: reward = 200.00, steps = 200\n",
      "12:51:52 [DEBUG] train episode 679: reward = 162.00, steps = 162\n",
      "12:52:10 [DEBUG] train episode 680: reward = 200.00, steps = 200\n",
      "12:52:23 [DEBUG] train episode 681: reward = 149.00, steps = 149\n",
      "12:52:36 [DEBUG] train episode 682: reward = 140.00, steps = 140\n",
      "12:52:54 [DEBUG] train episode 683: reward = 200.00, steps = 200\n",
      "12:53:06 [DEBUG] train episode 684: reward = 139.00, steps = 139\n",
      "12:53:25 [DEBUG] train episode 685: reward = 200.00, steps = 200\n",
      "12:53:42 [DEBUG] train episode 686: reward = 187.00, steps = 187\n",
      "12:54:00 [DEBUG] train episode 687: reward = 200.00, steps = 200\n",
      "12:54:15 [DEBUG] train episode 688: reward = 172.00, steps = 172\n",
      "12:54:33 [DEBUG] train episode 689: reward = 200.00, steps = 200\n",
      "12:54:51 [DEBUG] train episode 690: reward = 200.00, steps = 200\n",
      "12:55:04 [DEBUG] train episode 691: reward = 142.00, steps = 142\n",
      "12:55:21 [DEBUG] train episode 692: reward = 180.00, steps = 180\n",
      "12:55:35 [DEBUG] train episode 693: reward = 162.00, steps = 162\n",
      "12:55:54 [DEBUG] train episode 694: reward = 200.00, steps = 200\n",
      "12:56:11 [DEBUG] train episode 695: reward = 200.00, steps = 200\n",
      "12:56:25 [DEBUG] train episode 696: reward = 157.00, steps = 157\n",
      "12:56:43 [DEBUG] train episode 697: reward = 200.00, steps = 200\n",
      "12:57:01 [DEBUG] train episode 698: reward = 196.00, steps = 196\n",
      "12:57:19 [DEBUG] train episode 699: reward = 200.00, steps = 200\n",
      "12:57:31 [DEBUG] train episode 700: reward = 139.00, steps = 139\n",
      "12:57:48 [DEBUG] train episode 701: reward = 185.00, steps = 185\n",
      "12:58:06 [DEBUG] train episode 702: reward = 200.00, steps = 200\n",
      "12:58:24 [DEBUG] train episode 703: reward = 200.00, steps = 200\n",
      "12:58:36 [DEBUG] train episode 704: reward = 136.00, steps = 136\n",
      "12:58:51 [DEBUG] train episode 705: reward = 161.00, steps = 161\n",
      "12:59:09 [DEBUG] train episode 706: reward = 200.00, steps = 200\n",
      "12:59:27 [DEBUG] train episode 707: reward = 200.00, steps = 200\n",
      "12:59:43 [DEBUG] train episode 708: reward = 173.00, steps = 173\n",
      "13:00:01 [DEBUG] train episode 709: reward = 200.00, steps = 200\n",
      "13:00:20 [DEBUG] train episode 710: reward = 200.00, steps = 200\n",
      "13:00:39 [DEBUG] train episode 711: reward = 200.00, steps = 200\n",
      "13:00:55 [DEBUG] train episode 712: reward = 162.00, steps = 162\n",
      "13:01:09 [DEBUG] train episode 713: reward = 148.00, steps = 148\n",
      "13:01:27 [DEBUG] train episode 714: reward = 200.00, steps = 200\n",
      "13:01:46 [DEBUG] train episode 715: reward = 200.00, steps = 200\n",
      "13:02:05 [DEBUG] train episode 716: reward = 200.00, steps = 200\n",
      "13:02:24 [DEBUG] train episode 717: reward = 200.00, steps = 200\n",
      "13:02:36 [DEBUG] train episode 718: reward = 123.00, steps = 123\n",
      "13:02:52 [DEBUG] train episode 719: reward = 169.00, steps = 169\n",
      "13:03:10 [DEBUG] train episode 720: reward = 200.00, steps = 200\n",
      "13:03:29 [DEBUG] train episode 721: reward = 200.00, steps = 200\n",
      "13:03:47 [DEBUG] train episode 722: reward = 194.00, steps = 194\n",
      "13:04:05 [DEBUG] train episode 723: reward = 200.00, steps = 200\n",
      "13:04:17 [DEBUG] train episode 724: reward = 129.00, steps = 129\n",
      "13:04:32 [DEBUG] train episode 725: reward = 155.00, steps = 155\n",
      "13:04:51 [DEBUG] train episode 726: reward = 200.00, steps = 200\n",
      "13:05:10 [DEBUG] train episode 727: reward = 200.00, steps = 200\n",
      "13:05:23 [DEBUG] train episode 728: reward = 146.00, steps = 146\n",
      "13:05:39 [DEBUG] train episode 729: reward = 170.00, steps = 170\n",
      "13:05:55 [DEBUG] train episode 730: reward = 165.00, steps = 165\n",
      "13:06:14 [DEBUG] train episode 731: reward = 200.00, steps = 200\n",
      "13:06:32 [DEBUG] train episode 732: reward = 200.00, steps = 200\n",
      "13:06:50 [DEBUG] train episode 733: reward = 200.00, steps = 200\n",
      "13:07:09 [DEBUG] train episode 734: reward = 194.00, steps = 194\n",
      "13:07:29 [DEBUG] train episode 735: reward = 200.00, steps = 200\n",
      "13:07:45 [DEBUG] train episode 736: reward = 163.00, steps = 163\n",
      "13:08:02 [DEBUG] train episode 737: reward = 170.00, steps = 170\n",
      "13:08:20 [DEBUG] train episode 738: reward = 200.00, steps = 200\n",
      "13:08:38 [DEBUG] train episode 739: reward = 198.00, steps = 198\n",
      "13:08:56 [DEBUG] train episode 740: reward = 200.00, steps = 200\n",
      "13:09:15 [DEBUG] train episode 741: reward = 200.00, steps = 200\n",
      "13:09:34 [DEBUG] train episode 742: reward = 200.00, steps = 200\n",
      "13:09:53 [DEBUG] train episode 743: reward = 200.00, steps = 200\n",
      "13:10:04 [DEBUG] train episode 744: reward = 122.00, steps = 122\n",
      "13:10:18 [DEBUG] train episode 745: reward = 145.00, steps = 145\n",
      "13:10:36 [DEBUG] train episode 746: reward = 200.00, steps = 200\n",
      "13:10:54 [DEBUG] train episode 747: reward = 200.00, steps = 200\n",
      "13:11:12 [DEBUG] train episode 748: reward = 200.00, steps = 200\n",
      "13:11:31 [DEBUG] train episode 749: reward = 200.00, steps = 200\n",
      "13:11:48 [DEBUG] train episode 750: reward = 200.00, steps = 200\n",
      "13:12:06 [DEBUG] train episode 751: reward = 200.00, steps = 200\n",
      "13:12:25 [DEBUG] train episode 752: reward = 200.00, steps = 200\n",
      "13:12:42 [DEBUG] train episode 753: reward = 200.00, steps = 200\n",
      "13:13:01 [DEBUG] train episode 754: reward = 200.00, steps = 200\n",
      "13:13:19 [DEBUG] train episode 755: reward = 200.00, steps = 200\n",
      "13:13:19 [INFO] ==== test ====\n",
      "13:13:38 [DEBUG] test episode 0: reward = 200.00, steps = 200\n",
      "13:13:56 [DEBUG] test episode 1: reward = 200.00, steps = 200\n",
      "13:14:15 [DEBUG] test episode 2: reward = 200.00, steps = 200\n",
      "13:14:34 [DEBUG] test episode 3: reward = 200.00, steps = 200\n",
      "13:14:52 [DEBUG] test episode 4: reward = 200.00, steps = 200\n",
      "13:15:10 [DEBUG] test episode 5: reward = 200.00, steps = 200\n",
      "13:15:29 [DEBUG] test episode 6: reward = 200.00, steps = 200\n",
      "13:15:47 [DEBUG] test episode 7: reward = 200.00, steps = 200\n",
      "13:16:05 [DEBUG] test episode 8: reward = 200.00, steps = 200\n",
      "13:16:24 [DEBUG] test episode 9: reward = 200.00, steps = 200\n",
      "13:16:42 [DEBUG] test episode 10: reward = 200.00, steps = 200\n",
      "13:17:00 [DEBUG] test episode 11: reward = 200.00, steps = 200\n",
      "13:17:19 [DEBUG] test episode 12: reward = 200.00, steps = 200\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "13:17:35 [DEBUG] test episode 13: reward = 179.00, steps = 179\n",
      "13:17:46 [DEBUG] test episode 14: reward = 119.00, steps = 119\n",
      "13:18:05 [DEBUG] test episode 15: reward = 200.00, steps = 200\n",
      "13:18:24 [DEBUG] test episode 16: reward = 200.00, steps = 200\n",
      "13:18:43 [DEBUG] test episode 17: reward = 200.00, steps = 200\n",
      "13:18:56 [DEBUG] test episode 18: reward = 140.00, steps = 140\n",
      "13:19:16 [DEBUG] test episode 19: reward = 200.00, steps = 200\n",
      "13:19:24 [DEBUG] test episode 20: reward = 86.00, steps = 86\n",
      "13:19:43 [DEBUG] test episode 21: reward = 200.00, steps = 200\n",
      "13:20:04 [DEBUG] test episode 22: reward = 200.00, steps = 200\n",
      "13:20:24 [DEBUG] test episode 23: reward = 200.00, steps = 200\n",
      "13:20:44 [DEBUG] test episode 24: reward = 200.00, steps = 200\n",
      "13:21:04 [DEBUG] test episode 25: reward = 200.00, steps = 200\n",
      "13:21:24 [DEBUG] test episode 26: reward = 200.00, steps = 200\n",
      "13:21:44 [DEBUG] test episode 27: reward = 200.00, steps = 200\n",
      "13:22:03 [DEBUG] test episode 28: reward = 200.00, steps = 200\n",
      "13:22:23 [DEBUG] test episode 29: reward = 200.00, steps = 200\n",
      "13:22:42 [DEBUG] test episode 30: reward = 200.00, steps = 200\n",
      "13:23:02 [DEBUG] test episode 31: reward = 200.00, steps = 200\n",
      "13:23:21 [DEBUG] test episode 32: reward = 200.00, steps = 200\n",
      "13:23:41 [DEBUG] test episode 33: reward = 200.00, steps = 200\n",
      "13:24:00 [DEBUG] test episode 34: reward = 200.00, steps = 200\n",
      "13:24:20 [DEBUG] test episode 35: reward = 200.00, steps = 200\n",
      "13:24:39 [DEBUG] test episode 36: reward = 199.00, steps = 199\n",
      "13:24:56 [DEBUG] test episode 37: reward = 182.00, steps = 182\n",
      "13:25:15 [DEBUG] test episode 38: reward = 200.00, steps = 200\n",
      "13:25:34 [DEBUG] test episode 39: reward = 200.00, steps = 200\n",
      "13:25:53 [DEBUG] test episode 40: reward = 200.00, steps = 200\n",
      "13:26:13 [DEBUG] test episode 41: reward = 200.00, steps = 200\n",
      "13:26:32 [DEBUG] test episode 42: reward = 200.00, steps = 200\n",
      "13:26:51 [DEBUG] test episode 43: reward = 200.00, steps = 200\n",
      "13:27:04 [DEBUG] test episode 44: reward = 131.00, steps = 131\n",
      "13:27:23 [DEBUG] test episode 45: reward = 200.00, steps = 200\n",
      "13:27:35 [DEBUG] test episode 46: reward = 121.00, steps = 121\n",
      "13:27:54 [DEBUG] test episode 47: reward = 200.00, steps = 200\n",
      "13:28:13 [DEBUG] test episode 48: reward = 200.00, steps = 200\n",
      "13:28:32 [DEBUG] test episode 49: reward = 200.00, steps = 200\n",
      "13:28:52 [DEBUG] test episode 50: reward = 200.00, steps = 200\n",
      "13:29:11 [DEBUG] test episode 51: reward = 200.00, steps = 200\n",
      "13:29:30 [DEBUG] test episode 52: reward = 200.00, steps = 200\n",
      "13:29:44 [DEBUG] test episode 53: reward = 141.00, steps = 141\n",
      "13:30:03 [DEBUG] test episode 54: reward = 200.00, steps = 200\n",
      "13:30:23 [DEBUG] test episode 55: reward = 200.00, steps = 200\n",
      "13:30:42 [DEBUG] test episode 56: reward = 200.00, steps = 200\n",
      "13:31:01 [DEBUG] test episode 57: reward = 200.00, steps = 200\n",
      "13:31:20 [DEBUG] test episode 58: reward = 200.00, steps = 200\n",
      "13:31:39 [DEBUG] test episode 59: reward = 200.00, steps = 200\n",
      "13:31:58 [DEBUG] test episode 60: reward = 200.00, steps = 200\n",
      "13:32:18 [DEBUG] test episode 61: reward = 200.00, steps = 200\n",
      "13:32:37 [DEBUG] test episode 62: reward = 200.00, steps = 200\n",
      "13:32:56 [DEBUG] test episode 63: reward = 200.00, steps = 200\n",
      "13:33:16 [DEBUG] test episode 64: reward = 200.00, steps = 200\n",
      "13:33:32 [DEBUG] test episode 65: reward = 159.00, steps = 159\n",
      "13:33:51 [DEBUG] test episode 66: reward = 200.00, steps = 200\n",
      "13:34:11 [DEBUG] test episode 67: reward = 200.00, steps = 200\n",
      "13:34:30 [DEBUG] test episode 68: reward = 200.00, steps = 200\n",
      "13:34:49 [DEBUG] test episode 69: reward = 200.00, steps = 200\n",
      "13:35:09 [DEBUG] test episode 70: reward = 200.00, steps = 200\n",
      "13:35:30 [DEBUG] test episode 71: reward = 200.00, steps = 200\n",
      "13:35:50 [DEBUG] test episode 72: reward = 200.00, steps = 200\n",
      "13:36:10 [DEBUG] test episode 73: reward = 200.00, steps = 200\n",
      "13:36:30 [DEBUG] test episode 74: reward = 200.00, steps = 200\n",
      "13:36:49 [DEBUG] test episode 75: reward = 200.00, steps = 200\n",
      "13:37:09 [DEBUG] test episode 76: reward = 200.00, steps = 200\n",
      "13:37:28 [DEBUG] test episode 77: reward = 200.00, steps = 200\n",
      "13:37:48 [DEBUG] test episode 78: reward = 200.00, steps = 200\n",
      "13:38:08 [DEBUG] test episode 79: reward = 200.00, steps = 200\n",
      "13:38:27 [DEBUG] test episode 80: reward = 200.00, steps = 200\n",
      "13:38:47 [DEBUG] test episode 81: reward = 200.00, steps = 200\n",
      "13:39:06 [DEBUG] test episode 82: reward = 200.00, steps = 200\n",
      "13:39:26 [DEBUG] test episode 83: reward = 200.00, steps = 200\n",
      "13:39:45 [DEBUG] test episode 84: reward = 200.00, steps = 200\n",
      "13:40:05 [DEBUG] test episode 85: reward = 200.00, steps = 200\n",
      "13:40:24 [DEBUG] test episode 86: reward = 200.00, steps = 200\n",
      "13:40:43 [DEBUG] test episode 87: reward = 200.00, steps = 200\n",
      "13:41:01 [DEBUG] test episode 88: reward = 200.00, steps = 200\n",
      "13:41:18 [DEBUG] test episode 89: reward = 170.00, steps = 170\n",
      "13:41:38 [DEBUG] test episode 90: reward = 200.00, steps = 200\n",
      "13:41:58 [DEBUG] test episode 91: reward = 200.00, steps = 200\n",
      "13:42:17 [DEBUG] test episode 92: reward = 200.00, steps = 200\n",
      "13:42:30 [DEBUG] test episode 93: reward = 141.00, steps = 141\n",
      "13:42:49 [DEBUG] test episode 94: reward = 200.00, steps = 200\n",
      "13:43:08 [DEBUG] test episode 95: reward = 200.00, steps = 200\n",
      "13:43:28 [DEBUG] test episode 96: reward = 200.00, steps = 200\n",
      "13:43:44 [DEBUG] test episode 97: reward = 162.00, steps = 162\n",
      "13:44:03 [DEBUG] test episode 98: reward = 200.00, steps = 200\n",
      "13:44:22 [DEBUG] test episode 99: reward = 200.00, steps = 200\n",
      "13:44:22 [INFO] average episode reward = 193.30 ± 20.35\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO29d5wcR5k3/n16ZnZXGxR3lbMVbFmWJVnOOUcw+DDIvHAmnQ0272HMCzYHvOQ7H/EIB5w50r1kMAb/DGfwGSeCg5yEZFuOsq1g5Sztanenfn9Md091dVV1VXfP7Oyovp+PtDPd1VVP93R/6+lvPfUUMcbg4ODg4NBc8IbaAAcHBweH/OHI3cHBwaEJ4cjdwcHBoQnhyN3BwcGhCeHI3cHBwaEJURxqAwCgu7ubzZw5c6jNcHBwcBhWeOSRR7Yyxnpk+xqC3GfOnIkVK1YMtRkODg4OwwpE9JJqn5NlHBwcHJoQjtwdHBwcmhCO3B0cHByaEI7cHRwcHJoQjtwdHBwcmhCJ5E5E04jobiJ6iohWE9H7/O1jiehOInrW/zuGO+bDRPQcEa0hovNreQIODg4ODnGYeO4DAD7AGDsCwAkAriWiBQBuBHAXY2wugLv87/D3LQdwJIALAHyDiAq1MN7BwcHBQY7EOHfG2EYAG/3Pe4joKQBTAFwK4Ay/2A8A3APgBn/7TxljfQBeJKLnABwH4K95G+8wNHj61d3Y2zuAZTPHAgCe3LAb9z+7BW87eSZai8n9+Cvb9+P5LXtxxvzxuPWxdTj/yIlobyni1V29eOzlHXhh6z7MHd+JzrYiJo5sQ2dbEQ+8sB2vPXoyAOC+Z7bgsZd34k3HTsOEka34xSPr8LrFU9BSrPgq+/oGcOeTm3DekRNwx6pX8folU0BEAADGGH74wEvYvKcPR04ehZ6uFhwzo3Iez2zagx37DuL42eNw7zNb8JfntqJU8OAR0DtQRkvBQ6ngYfLoNjAGbNrdi70HB1AgwqWLp2DV+l24bGmlrX19A/ivv76EcxdMwJzxnegfLOPWR9fjDcdMxYMvbse4zhY8/vJOtJY8nHjYONz/zFa8tG0f9vQNYHxXG6aMGYGnNu5G0SMsmzkWp8+rzFP53d824oUte/HWE2biN0+sx9a9B7Gntx+Lp43G81v2AQBmd3dg855enDKnB3c9tQkXHjUR2/f146EXt2Ha2HY8/spOdLWVcM4R4zGruwN/fHozLl08JfF327S7F79Y8QpOPKwbrUUPtz62HiPbSpgyZgTKjGHG2Hbs2H8QBwcZXtm+HxNHtmHNpj04bW4Ptu3rw8p1u9DRWsSe3n5csmgSVq7bhYVTRmH1+l0Y09GCrrYi+gcZVq/fhTID5k/swrOb9+KSRZOwfscBPPbyDgwyhhGlAi48ahKe2rgbr+7qxc79/ehoLeLUud34w5ObQADmTejCjHHtWLF2O7bvO4hZPR14ccs+zJnQhaXTR+MXK9ZhsMzQUvQwb0IXntu8B3PGd4X39qgRJZx1xHjc/fRm7Nzfj/kTu7Bzfz+6u1rw9Kt74BFhVncHNu/uBRGhu7MFL2zdh6OmjMJL2/bjjPk9+MPqTRgol7Fw8ig8/spOtBY9zBjXgXU7DmCwXA6v67yJXbhk0WTDp88cVpOYiGgmgCUAHgQwwSd+MMY2EtF4v9gUAA9wh63zt4l1XQXgKgCYPn26rd0OQ4gL/u1+AMDamy4GANxwy0r8bf0uLJ42GsfPHpd4/Llfvhe9/WXc8p4T8f6fPYE3HLMNX7j8aPzdN/+C9TsPxMpftnQKfvXoekwbMwJLpo/BJ25bjRe27kOpSJg+th0f+uVKrNu+H9efNx8A8PHbVuOXj6zDvAmdeGbTXkwZPSK0a/OePnzsN6sj9Qfncd6X7wMAfO/tx+Lt33vY6pp8457nAQCTRrXhpDnd+Mvz2/CvdzyNh9dux3ffdiz+668v4dO3P4mDg2V89NerIse+fskU3PrYem39L/7LRSAiXPOjRwEAnkf43B1rEqx6GgDwxTufke59auNulAqE3/3tVcyb0IUjJo3U1nb5t/6Kl7fvB/AMZnV34MWt+xLar+Dm+16Ibfven9dKy/Z0tWLLnr7Itld3HcBDL27H2m37w21f+EP8nH780Ai8sj1+//BoLXq4+vTD8NW7nk20+7t/fhE79vcnlpPhxw++LL2Xefj+Bi5ZNHloyZ2IOgHcAuA6xtjuwBOSFZVsi60Iwhi7GcDNALBs2TK3YsgwxqbdvQCAwbLZz9jbX/Fa9vYNRo5XPQztLZW3gUde2oEl08eg7C8w0z/AsKd3AECFtAME5PDMpr0AgH0HB8J9/YNVj0kFkdinjB6R+KAG2NNXaWvAb2fn/oMAEHpqL2yJE+K6HRXSmjpmBNbtkLezZW8fxne1hd8PHBw0skeHwTLDZv/a9/Yn11ch9grWbjMjdlsMSH6fgTJD/2DyvdXbX0ZPVyt2H+hH34D8d+4fLEe8Zh0OGFwTtS36Y5/97IUoFWobz2JUOxGVUCH2HzHGfuVv3kREk/z9kwBs9revAzCNO3wqgA35mOvQiAj6+Vr10EVPfpsOMoaCV2l8gOtYSoWof1EuAw+v3V6xMYWRaj9GDbGf6+lqBVAhaREPr90BAPA0DYkeqQnZJYExFv5mGmdNcWzm5qWQ+QembQXORXBPqOo39EG0v0cSZE7E7J6OXOo2hUm0DAH4DoCnGGNf4nbdBuBK//OVAH7DbV9ORK1ENAvAXAAP5WeyQ6PC5CEMvHQbBA9t4LGzcHsZRf9BHoyQe/S2/spdz+Lyb/0Vj7y0I6zDBqoHccroEbFtQfVBOwFpjmwrAQC27FGfv46UNu/uxeoNu8LvMg/XFgzAynWVOj0CXtiyF3v7BvQH1Riy36fMGEyWAx0sMxCAQgJx5nHtTGzhsWzGGBw+sSv8rvmpc4OJ534ygLcCOIuIHvf/XQTgJgDnEtGzAM71v4MxthrAzwE8CeAOANcyxrK/Qzo0LMhX4piB7378P99lXf+A/xodPN/B34GyynOP3tZPbtwNANi+76Cx18ZD9SC2FmWPT7QjEg/dvu+gsh0dJz2xbhcu/uqfwu8DaU5EwF+e3xZ+9ohw1hfvxf/69gOaI2oPGYebetvlMgMRUCjomdP0rSeNIxC2IRhc8CjydmT7ppQGJtEyf4JcRweAsxXHfBbAZzPY5TCMENynOfCNFAODUY89eOgGB1ko2fA6alF4uAMvqlgg5QNb1hjvKdhdtl3sgELJyv+u03F1HucGQfMfMNSNdTjI6dJB00+s26UoXR8oPXcDx2GQMRAo2XM3vHamY0jSNoS3g2KBQhKtA68DcDNUHXKEyaszD9N7PCbLyDx3zhsrKTT6kucpbdx7UC1HqGQZHYlUPfdomWAw2aYdID6eMZCD5s5DtFNapg6kJCN3xpiR4xDcJ6rOOIDptcvydiQe6lHVc6+H3g44cnfIAcGtWqsB1eAhE/XsQY7ceVIQPXd+u+p5ZRpnTsUVsme0+nYh396riXLRkZLYKR3MWTdW9Id1h+z3KZfNxnPKrCLLFBPI3VSWyTpo3NPVGtpS8KrdZz30dsCRu0OeqBG7i6/HAZEPlFn48Og09+p2tSyje+1Xeu7aqIxAl/Hr97/3DmhkGc3TWGvP3cSbrAcnyd6sGOwGVJPOJQ9JywQFIhzW0xl+Dm6XeujtgCN3hxwQzv60ZPegdNJzG4SVBQ94wOOD5XJIsNFoGYXn7nlQPdc6G2zIvaq5RwdUg+p1XqNWKxYOy5ugTLzJepCSTOcuMzO/ocwqNuo6XSCfwWgTFDwK38b4AVXnuTsMO9i+xuoGMXlUNfegHYnnzpFmUeEC6wZUdVBJFjKyY4jaasOHWllGoLc84twbEfI4dzPPPUASuf925UZbs1LB86pEzssyJuMbubRfl1YcDgnY8qapByVq7iz03JnCc5ff1rqHSmeJekBVUo8Y5x6EiRqcqk5O+N3fXo18zztWu1aTkvKAzcQjIJnc64UC9xbheRS+xtXLvIZYINtheOCDv3gCr2omIdnyg+k08IC4mRBDPlBmoffMyxQlEy1cgM4zVMkROjKOk1Hy1UkK4eORt7RgUt1QUabpJCag8qZkcx1rCc+j8B4pcp/rFS3jyN3BGL94ZJ12v20opClBVTX3yvfgsP6BMt74H5Vko7xKoZJlGMxC6kSo+gppnHvwN5yhGnxPbsfmmTfJkWMDE7lqqDizzMzfLIiSQyHrBY8bRC2Qi3N3GIZIm1tG9MhVqMoyUc+dnxDEvwWoBlT5Y0XoLFB5gto493KU3E1gIyfkHS2TZTZmrcHnwEkCgZShkLr7ohYQZZngXqhX5+PI3SE32PKD6QzAQF8Wo2t479WE7BhTv12kiZaRDbSKET3V1AzJsCF3cXp7VjQwt6PMmFXnoyJPVQK6WkGUZYJ7oV5djCN3h8yocl9tZJmDQfoBySSmAPxnHQ+k4USV963X3O1lGRstNu8BVZOONo8oDz4zoilMJzEBgeYu36ea3FYrFLzqb+p5FDoDboaqw7BDrT33MmO46b+fxn5/licfDhghd00nowq/1B2j8qhlD6kY0WNSf1I7MgxXWWbx1NE46/DxyQU5mOaWASpeseo6ttQ4f7ooB/GyTEXC8z13R+4OwwU20gOPwHNP8ggD+WXr3j58697nueM5WYYjbWWKAV1InaUs85qjJ+MjFx+hPCZdamHzsv05T2Iy6mct7Dth9tg8qgGQ8LuJdWsmMdV6cQyxXSJxElNluwuFdBh2sPbcQy3dbEBVfMAHVJ67pjrbiB5ALst8dfli7O6NJxuLT2LKJ85dRCN77uccMQEjWtRr6do6rmXGjD0Hnedea1mmVPAiK0AVPIpOYvLH/50s4zDsYJt+wJSfVFIHnzzLRJbRhULqTJG9RhMX5iaDmM/d5FSHUnPn5aqnX92Nz9z+JLYKq0aZWlfw9HMNbH136wFVxXWst+deoGr64QI3uFqvUEjnuTukxuoNu7Bzf7/VoCEPU4KqpvqNNhCJljEeULWPllE5fNL0A2FHVNtQyLyjZQa5C/D1Pz6H21duxMIpo/C6JbG17RPhGeR3sYFpbhkAAKmzQtY6FFKs3/OgkGXcJCaHBkewMtDMce0A7F/tTaNlqvHwUfQP8LJMleh14Y55LrMn2ypOtEqyKdKO1YBq7dIPBAPW4rUy5SQtuZO958osPHedLFNrz10MtYzIMo04iYmIvktEm4loFbftZ9ySe2uJ6HF/+0wiOsDt+1YtjXeoHTbv6cUDL1SXYdORSdrR/wEhxFEFMWFYAN5z37G/H9t8GUFVH9PuUxthk34gqEW1zJ4ONo5lLTV3xuS/i2kopCcsKSfCekBVYouybqKGkWX4To6/Jo2kuX8fwAX8BsbYmxhjixljiwHcAuBX3O7ng32MsXfnZ6pDPXHp1/+M5TdX19M0WRzCPhQymlZAXa98QFW06Qp//U9ddalkGWVWSF075vUHsPHc+7l0x3mAt1fsoGzhkSY3PUv2XMX9Nsvd6T332pKqOGDLk3iBGjBahjF2HxHNlO2jSlf0RgBn5WuWw1Bj465ogrA+zfJw1UHDdLJMEokEerBYSvRen9m0t1JOGQqZNreMQpbRPKRMYXOadmQYLDMUiDCY0wop0uXt/L+7e/uxfseB2H4VChQdPLR9A/CIImMAtmuZKqNlajxDNRbnzn2vrKFa3zj3rJr7qQA2Mcae5bbNIqLHAOwG8FHG2P2yA4noKgBXAcD06dMzmuFQa5gkqkqb8jfp4Q0iOXSyDMAv1K2ZxJQit4xac5cNqEY7rHCANeeskGWmX7nJFmXJgHRwLm//3sN45KUdGFFShzfy4KWRAhEGLLV7jwB+vSqbNwgijederK/mzt83Hue5N4zmnoArAPyE+74RwHTG2BIA1wP4MRGNlB3IGLuZMbaMMbasp6cnoxkOtYaWf1NHy5h5t1WCjGJQJA3Iy4Hbrh5sVVuhkktkm8uChCSSvA42jmWZsVxzlMhkmcDmR17aAcA8E6XHEazs2iXLMsR9rjoBRqtFgZSdZEudZZmCV72GBW7hjkbS3KUgoiKAywD8LNjGGOtjjG3zPz8C4HkA87Ia6TD0MPE8bQWCquae4LkrQiGVh2mjZexsBHQLZMd3iKSeYJLQjvlDzwy0axvIBlSDc2krVWjCNLpJjBIRkSzLVD+XPI/LsJl8wjrPveayTCEeLVMld29YLbN3DoCnGWNhkm8i6iGigv95NoC5AF7IZqJDI8CEnGxnf4ZRMAnlBg293+parro27QdUbUIhY3KMxSWxHSDNc7k21cLUAIzlmABE1ciQ2DkREsNl+OtdLJCV5y5t00dessyFCyfi629eEtsuau58Z1Sg6mk3jOdORD8B8FcA84loHRG909+1HFFJBgBOA7CSiJ4A8EsA72aMbc/TYIehgU73TJJDVDAl7bIhUQbPltrWdAOqqmdRtj303IVc9SbN1uuhl4FXXMRY/TZLcvciRBbfn3SWkSgTj6qeu0Uopgz6WbNVfO2KOHHzGN1eQldbKbZdljgs+P0L/EWpE0yiZa5QbH+bZNstqIRGOjQZjDxQS+KsEmBS22ZRNSbrlabJLaOOltENqAbfzdu1Jfc8+wJ+/CKU4Fg6z50nspgXzZLlFX53qeCFtpmcL5FusQ4zz33CyLbENmRyk6i5P79lLyaPHgGgIsvUe5k9l1vGITOqcoitLGPmkqtmqMYNgbYcY7owSXW1NmpJWQjvDGUag2Nto1/ypAi+8xE991Zrz70a9ldIoXNHZBmPwt/fiNyhJs9S0eyKJbVTIJIOfovnunrDbvT6q4V1tBaqbzN1Yl1H7g5GMHF4n9q4BzNv/C0efXmHUZ2BFJBUta0skyq3TIp87vL6AxvMo2QC2C6/lme8dFlC7sE5BAOq5nZBG/aXZDV/vUsFLyR3E483jwHVpJ/BI7ktsjeGA34qh662Yt1zyzhyd9CiOhknWXO/Z81mAMDvV71qVLdNmKBvjXZvVZZREXjalZhsyF2QZfjGE2D70O/ti6cclqFdk343AJ8eXkxb3Fa0lGVIr46bxLkH4AdUTa+OcrEOwwHVpN/b8+S5c2Tb9vnk3tlaqvskJkfuDlrYRH3wRcplhmWfuRO/fGSdsnzZoOOIltfvD7NTquxjaaNlkm0LIMoxoXyd8yQmG5hMApJ67v53W8+dfwOJnZHBuCIJskx+oZCGskzCfk+R7llW//6DlQ64s7UYHlOvcVWXFdJBCyb81ZblCu3YfxBb9x7EZ377JN5wzFRp+VBLN/SmeY+8wGmxAQLPVzd4qfPqVbDxqKtx7kG95udoK8sY22Qw9yhC7v7fT9/+JEoFsk64lSjLJHnGqgFVk7ahzrNvSu5Jvzefmz1ynJTcq7JMcDGGQ5y7wyGAqnaskWVCj7laZoufobGns1V5nK0sw3O5fHIMtPXlnVtGBnHClU17tfLcxZm8Mtxwy9+qX7ji//c3q63JvcANqMpgHwrpH2cULWMX3aSqI2m/7O1A7Dx+/A/HY39f1XMP9jpZxqEhYOK5y0IQN+/2yb1LTe7GUTA+eO9SNzamI1RpgizGpJ3XDD9Pvc2zWI00iZK8yTnWyqOzTbwlwnpyFSVcs0TNnZNlOM/d5M2GYJeiOU05PjGauJ3HkZNHheU6WqsDqk6WcWgImMRph547p9Vu3pNM7qo87Um2AAovV/IGETkecuJnTE6+QRs2M0GroZDVNoM2klArWcYW4vWzXXuU99tj145ZxrnzoZDG7cshXl6P5PdD8oCvgtyFBooe4efvPhF/fHozWorVOPfhkjjMoclhoxnzCAaS2lvU/kPZ0qOMeu4SzTPU3OXHV+Lc4ztVA45BGzYP42Aox9hftzzzs2eBaLOpVh2B5qKZkGfYNtexGHneCuKVtfuba0+Rl0vKfeORNDd8bA1Vj3DEpJG49sw5fr31hSN3By1UGRnlZeMRFzpeME0/EIAnYRkRmjz7sg6FKWyoeu7miM1MDdtIPskG4faYpSqyO22eOpur8lQMomXEAdXwUM2B8yd0he2q00VEdxw1dRTefHw83XhSOLxH8rBK0eGI55rR15s3HLk7GME6FNJgyrgYUZIE0wFV3UQllSwj677SeO7xfO4W0TJDmFuGh/h2o7qe1587D6fM6dbWJY+W0bcvzlDljkxspzKgmlyvrsak36FU8KTkrluso2KbvcyXBY7cHbRgBgRMEjkkIFHdjVzOEAqZRpap2GUhy4SEYR8to/LgdWgYWSbhewC9l1wtYwu+Tn5Kv+7y8NdOPaCqbyvclmBfwSO0SCKIVGQea8tp7g6NgDSaOx99onsgB4J87sb1Vj9LPfeEAVUwu9wyacg2tkiHRb6dhiF3wWR15ydfCLu7syUhFFJ/nnx0D69t6/pYj5PQVOVkHvlbT5gZm8Gb1JkXPTLy3EXUy2MP4MjdQQu7GarVQlVZRuO5W0bL8PHaciJM57mromXSyCRKOcbgHOsV/5yEmOeuMJ0E/fyyJVMwrqMFr18ylZNJ7MdG+rm1cfkFMHS/R1VCMx9QBYD5E7vw5KcuiER18eXedlKc/AsKck+KdnKhkA6Z8NK2fdjbN4AjJ4/KpT6TAcHgZpXJMjqYpvyV1Skb9AoeHlXbqlBIlWeaynMPQiGDpGg2skyjkLtwPVSdLx/P/snXHokrT5pZ3aepP+k0B7gptXwOdt1h/NR+m5WzZHXzncPHX7MA7zxlFk793N3htqJKlrGYeVsPOHJvMpz++XsAAGtvujiX+mwGBG0HVAdtmE8wQjegqqpQlVsmKVrGJgd80HmkCYnMc8HrLBAnPak6Sz7eW/w59DynZ7mBiOeerKUHtgTtquQPU3Lly1VWlYru55fM4+FkGYdhBRNVQbzPGadt616ls3nuGs1dU6F0OTnGpG8msreD9pYC5ozvVNZfzZdjMUjho1FkGZHMVWdSydtegWi56lwuOHKigecul2V0x/GdvY3mLoNIwuK5qEjcWJZplAFVIvouEW0molXctk8Q0Xoietz/dxG378NE9BwRrSGi82tluIM9Xt3Vix37DlodYxItI5YFwC2Npoat5l5O9Nz1mvtL2/cpZBl5eZkss/qT5+PO95+WaGOalZjSyDJvXCZPypYF4lyApzfulpbjZRmTjunWa07CeUdOTPRfBwYVsozmwKqeLR/kBfSeO3+IeLh4XPask/VhdxPP/fsALpBs/zJjbLH/73cAQEQLUFlb9Uj/mG8EC2Y7DD1O+Je7sOTTd9odZDCgWn0Y+AFVcV8c9rllqp9lD1jSGqofuXWVfJ8iikYaF62IEAkQnJMYLVOrUMibLluET7xmgfVxOojX6NnNe6XlKubKZZkA0bBGfdkAh3FvRsYDqtxopfIyao7nTzkmMQlkrErHkJztssFkGcbYfQBMF7m+FMBPGWN9jLEXATwH4LgM9jkMMWzIqXqMabSMXZhlJM5dQbxB+yocHIjnv2VgxuSehOCc+HTG/YNlozemNLllPI+sF7BOgkn+dyCqR8eljOhfvozOcx3dXooEAxQNB1TDjgPq381ccyfhu36/KYbTDNX3EtFKX7YZ42+bAuAVrsw6f5vDMIVR4jCJHGIiQ8QWtjAsD+iTWemqk61epAv1q/w1fyoDUu/3pYUyY7jhlyvx1T8+l3hs2mgKk5S+NjBN+cMvWpHk7fJldJdzbHtL5Ltpbpm0M1TF42XlxN8/7XyEcHyiUTR3Bb4J4DAAiwFsBPBFf7vMbOmtQkRXEdEKIlqxZcuWlGY41BpM+GtSFjDT020X64gMqEqekGAAVDc+sPtAv6Re+YBqmmcwGAwM3hAYgNue2GB0bNpQyKwpfUUYe+7gvXFhn+RU+IlGpojkktcNqEZDXKRltJo7V3lS5I+qnqTfIUw/0MjkzhjbxBgbZIyVAXwbVellHYBpXNGpAKR3NmPsZsbYMsbYsp4edQIih6GFVSgkV8hEcrHlpEhWSMkDkjSgCgDbJPKIKhQyTfRKMBh50HT174ztAdHQwTxg2ll4nCwT83bDv3HStDnPVm79Vt1R1Y5DvRKTrt3WkjoqJ8mTD5B03aqx+I0zoBoDEU3ivr4eQBBJcxuA5UTUSkSzAMwF8FA2Ex2GEkzySUR12n8VsQWiJRBnc1oYo80KqasvWEQkUq1KljGzKoKAZ3nP3ZR6077u5+25m/4cfLSMeLF0nnt0m1hp9ePyY6dhRMl0QJWzSfPL/fzqE6Xb+UXATTX3ez94Bv50w5nh9sT7uM6ie+IkJiL6CYAzAHQT0ToAHwdwBhEtRuW+XQvgagBgjK0mop8DeBLAAIBrGWODtTHdoR4I7lcdfwS3LJ+qwCR00jZaZjDiuUvIPbBDU8fmPb2xbWli0lUY9GdXhuRuUXfaSUwDOZO7aWfBE2mco+MShIzbbnvvKbj76c344p3PxPYdNXWUVi7hwU9iUvWRG3YeUGax5BcBV51LgOB3mjGuI7I9cQF3/e7ckUjujLErJJu/oyn/WQCfzWKUQ+PAKHGYZCanSU4a26yQvCwje9Bl2SlF7Ngf19wZgEu+9idJff5+C4IeFDR3G95NG4XBx4XnAdMBWl6WUUkVFCkflK1u6+lqxTEzxkAFPoeLUSikptyB/kFlB8FHHMVkGKHTVZ1rcH++9ujJuPr02bH99Z7E5NIPOOhhMQknIsuEXrlmQNUycyJvgs5zNx0QlNUrr9EcAc8Gmns9skLW03MvehS2x2eFFE2XSXVhTDzvjSfYIkvQJUMwmE6K1UA+/4ZFuGDhROw/KBcSeHKPR/5EoRr4Du7nwyd1SXM7Vd8uGlhzd2g+/PCBl7D85r/GtptEy4i3Kr8ohsmAqrHnzpGOlAgJ2LHvIG5fudGswtAOuQFpnsGgrr5QljE/Nq3nnrfmrnsT4MmWOB6NR5jE5Rpp2CRBy/AtlpOYKrJMvNzly6ahq62kbCoqy4iau/57gOD+VJF/sNVySdrUcJ67AwDgo79eJd1uk/KX7wFMvOcsuWVk3o9HhD89t9WwtipU7ad5BmOhkPWQZTTkTmRnA6DvLCqhiYNh3SqTg04gEu0ifdsirSffWjTLLROVZWIMaicAACAASURBVNTlVD/qCM1EsBi5K1zi4LIlpRyuV95+R+4OWlRnqKofeJl3byLj2K6hytugcNxTLeasS2lb+WteZ7lcWajkYAodPPUkprK6rQIRBizZXddZiBq4avWrNy2bhg07D2Dh5FG49seP+uURK5t0aU0192pqg3gWRx6qSBpxlu/PrjoBG3f1Sm1Ueu4JmVCDtutF7k6WcdDia398Dn95bqvevZaENNpMYjL13ZMmMRGle3DUoZBpNHeG/sFqOoMyY8YDsmkf+n5NnHuQ0mDiyDbj+rTkLsgkYqRUWK7o4YYLDkdnWzFSHhA7aT0ZizKQCroZpqpyPERyP372OLxuiXxyvar+4Bqofsd6e+6O3B20+PGDL+PN//lgihmqyZJLIMts3XsQX//js8n189Eykv0eUa7kngaD5ajXXgkLNUOa3DJBmyoE+u+s7g7ccMHhmeuLkG34n520FfHcFWXC9grqEMWzDx8ffuZnv16wcKKyPhUxv/+ceRorolCFrA5yA80yVGeoOnJ3aCDoNPTQ/45o7sl18iF3X/hDPM6Zx5GTRyZq7kA6glRJTmmewUHGIsnJrKJlaqC5B51dwSO8+bjpWDJ9dKo2ApSEXC+B724yKK3LwV/9Ht0Q9dyj+276u0Xh50JInBWd/7Klcq9bdYVHtZdwy3tOwmVLpqCrLapWtxY9XH5MNbWycoaqfw1U92B1QNWRu0MDwSj9ABcTb5K2wDTI4/wjJ2BkWymaz11x56bR3FV2pCL3MsNPH345/G7zVpD2mb/mjMOUIYPB5fA8wqj2Er5z5bHpGvERIVuPs1lxnrwXKzs9Xf71WHsx7Zv7HIZC6qG7xsfMGIMvvWlxjJyJCJ+//GgsmloJb1SRs8mi8ICTZRwaDAaSewRiPnMZxEUhVCgWPBBFy8tefRlLq7krvM4wV405Q5cZw+fuWAOg4uVu3tNnTPBFz8PCKSON2wowbWw7fn3NydJ9wfXg1xjNgqIXlUnCQVLV24/ElrKgy+gIt1UzoBrpOETJI8dxlABJskswrq3aH3r2znN3aCQYTWKSyDK6w0xnQpa8SlRG0iQmIEo+pshxDlN0/U9LWzwPuP1/n2rfqAahLBNONspGLHznycsy6rTJ1fIlyetW0oShmMbPgfewRW86z980wAfPn4+RbUXM7umQ7g/uZ5VnH8bB14l1Hbk7YP/BeI5zEVrPXVjQ484nNxnFuRt7tIHnnpB+gIGlenDyTBwWyTlv+RaRxatUec6xWZEZncaIFMJFuigHVLnyQW52mwFVXVqAqC3V+nTIooicMX88Vn7ifHS0yiPIywmae0j+KRyQNHDk7oCP/Xp1ciENEYcPq/93/c4DWL/jQNJhxggG8fi61Otk5ifL2GLSqLZIpEnBcipiLd7WqwOq+bRBESmkWp9yQJX7HES+8B2RGJcu2tfRwhFpTHPn3iKEJfzUcxdqJ4kETao6EOe5O9Qdr+zYn1jGaIFsrkxvihmaKhQ9Dx5RhDjT5OxWQe112oWuFQsUrsAE2HvuWbxKldfvCXJMVllGXBMjKVkb78UG1yMqr+nb42Wgs7jQx8qxJPkcdwR41FLtLido6kmafd5w5O5graer9kXGR0V3PgOKBYpNo5fpmnyUjg3ykmWKnhfmlAm+2yH/hz4cUOXWGM2jPkCYxKQoLxtQje5PEmaqWDp9DNbedHH1WIksE0A9BmDUVCoE5K1qYzD07B25O9QJZqssqfeJi0ID+cgxAYpehQLE13kRadvMK3FYwaNIjLtundc82jNBNVomKlukhTK00eAayhYw1+WnEVESrqdWllHUUctVkMqhpq4fUE0TrpsGjtyHMbbs6cMBRQpTG5iQom0aXZNl9kxRGVA1k2XSNJef507o7R+MfLdBLR75MM49/JutFb5TLXhcbhmD8gHE602Rz2r7poweEfkenFN3Z0tsfdakfEG1QBAKmZQS2MW5OyTi2M/+D97wrb9krscossXy+HAJ0RzIvRIKmZxbhjGWqj31DNXkh3D1J88PPxcLFJFlbFGLwb5CTnJMWJ/giYcDqoo5C/J29VFPKvR0tUa+e0RY9cnzcf+HzpKnE5bZU0NePXbWWADxFZoCVGWb+pC7ywo5zLF6w+7MdWSVZWT7BlIsVqFCseBBpAmZnB0NyFSj4EXfAvb0ykNBk7xAAJGwuILnRVIBHMxA9LYY01ECAHS1FrGnr3o+YihknrIMgGTNPaHBpBmqADC6vYSd+/tj5YiATv/6h+kHEgdUa0es7zh5Js5bMAHTxrZL9zecLENE3yWizUS0itv2eSJ6mohWEtGtRDTa3z6TiA4Q0eP+v2/V0niHfGA2CKmZaSo5Ps/VgYIBVR5SQmBmHZX4Wrz85gfkBS2fwZJQ70FNtkYd0hDwpFEj8D/Xn46PXbIgsr3g5UPqAeIEmxAtI/uZ+Dh3Sr7Md3/gDDzw4bO1tsTuhyEYUCUiJbED1WcibYI4W5jIMt8HcIGw7U4ACxljiwA8A+DD3L7nGWOL/X/vzsdMh1rCiNp1nrtkWxASmI8s40lWoDe3RYSp52Tr5YmdxsGBdOMhaR/9OeM7Y4O4VVkmn1DIyaOjqYOTBjBl7UUmoxm0OaajBRNH6VMWmw4Y14dW5QicqIZJHMYYuw/AdmHbHxhjwbvfAwCmxg50qCv+8txWzLzxt6mO5QlY5cVrZ6hKdobknsqiKIoFihGtjDTaW9Wr6fBQaaIqmGqk4vR62wU7glayaLKq/CvB5lLBw6+vPRkfvfgI67p//K7jcfq8nsi2pPw7I1r0v0nSJCZTiJO0soyj1ArV3DL1aS+PAdV3APhv7vssInqMiO4lImWiDCK6iohWENGKLVu25GDGoY3frbJbN5QH70mpPG295i6RZQbNomVMnrWWome0Gs7McR1Gbwr/eNacWFhdkm3//b5Tcc//OUNbPu6525F7Hh2heFlkA6qLp43GyBEl67pPmtMtyZhY+au67l2SqfriJKY8dPDqmq32MlG9EPT1tjOX0yITuRPRRwAMAPiRv2kjgOmMsSUArgfwYyKSprljjN3MGFvGGFvW09MjK+JQJ/DyuIpgTJbZ49Fv6LV2KvJ08GgpxMldxEg/B7fJ+EFL0cM5R0xILMc3ecSkkZjZrff4+UWWAfOUxqp233PGYekq4KBKGJaWXkRJqzqgKj/ZzjYJufPH5+RJm4YXDqnn7sdK1kuWSR0tQ0RXArgEwNnMf6IYY30A+vzPjxDR8wDmAViRg60ONQKLeO4MskdfR1S6AdWkaJnO1qIyWiVAxXPXyzJEBAYz77fgkZGHb/sMtrfYP06tRS8WPhm0e7SfP9wGseukyCmTVnsXSTTJc5ctPB2Lc8+B60KzEuypBX51zUlG17Oz1Y9oarN/a0qDVORORBcAuAHA6Yyx/dz2HgDbGWODRDQbwFwAL+RiqUPNwEw8d91KTNJQSKav0Icqwx6P1qIX627E5EsewThaxjQtgK1ckKQvixjf1YqJo9qwct0upQW2iF8n+UBjam1b0cmmDYXMC7FJTLnOkdZj6fQxRuWuPn02OloLeOOy+gxRJj5ZRPQTAGcA6CaidQA+jkp0TCuAO/0f7wE/MuY0AJ8iogEAgwDezRjbLq3YoWHAPwipJgFJjjloOKBqJMsYe+4sPJeWgqcc0DR/hTcqFkLmpaapP0i2kEYfFutsK1ZsGhDCMtNybiylgv/VZgZzLYg3ltq4AdFWKuBdp86uW3uJTxZj7ArJ5u8oyt4C4JasRjnYQ3y2fv3YemzYdQDXnDEn8dio5q6IlrEcUDWFmeZeiHmksXjr0JjKn1PmduOPT2+W1mea88VelrEkd5Vn7m9OI52IdQZvRr2C9JNelom+9ZT871ZvORpZhoiqkS8WdYr53INb8rKlU3DhwknmtjUR3AzVOuPpV3fjlkfW4Z8uOqKmXsZ1P3scAIzInZlEy2gnManrvvWx9fjYJQswtqNFun9cp3w7j5Zicpw7UVRH113ZgkdW3qNp55VGc9fZmWZNB/GWCsi9r9885n52dwde2LpPuk+UZa4+fTZ2HjiIK0+aoazvC5cfHVl0WryaIol/8LzDQSDlItcyiFE8QRvnLZiIcxckD543I1xumTrjzd9+EN++/0Xs2N8/1KZgw84D+JffPRWZiq+CfhKT/vjP/36Nct+4jlblvgAyWSY+QOin/A33q2mz6BHMgnnsOl/Rc//K8sVWx4utpgkRFDu9Tj/238Zz1/kcoqTV1VbCZ153lLZje8MxU3H+kRPVlQoY1V7Cp1+3MLIKUxJUk5gaWKWpORy51xl5rfqTB6772eP4j/tewNpt1cU60sW552yYgJZCfEA1rrlHbdHp1QWPjDTioE7TNyxxQHXp9DH4h1NnSctevGgSvvmWpVL2qbYb3T47IRTTPyryLSBd0XPn6778mOgAn474bdMYyyA+AxFZJmWd8ayQKStqIjhZZojQCCQvi0VXau6aetLGc49sKxrJI0YDqogOqOoIyiMyyn1jSzSi586nxBXx729emlhfmtSwYnOBTaLnzr8VfP7yo7Fqw248tbGShC7p2mVFTJbJwbuuzlCtxssAQ5tuYKjhPPc6I7j5tuztwy9WvDLE1sQR9DmxPEzazigdu3uG8eaySUyyxZIZFwqp06srS/Yl6zJZB1QLXnLGQ2m7oMhfu2PlNomeu24MQ5fYKo+MhrXwa0jw3MXthyIcuQ8R3vG9h/HBX67Elj190v27e/vxq0fX1dkqXZy7+pisMzGTUCrGaU42oAqYae4ewWicwZZcRwlT+j2ixHBGfvckPzmW0nQDc8TzDnRr1UQp2XeP4qseBchjoYmkAdU0EDv7BngxHnI4WWaIsMkndZX2+4nbVuNXj67HzO4Oo0kSed3L6sRhuklMtX2SSjLPXUIykWntmvrIlNwtOWd8VzRzoU6W4csAlcFXcQAxjdMpHjJC4bmLJcXFpttKBfQPxmcO11pzT4twUlsw9hL9ekjCee51hunNtvtA5cHavLs3dxvWvLoHL3ODqDxUD0UtPHePyKhjKBW8RJL0vECWSdbcATPN3RaxpF0GnrsszwgJf8XtNjb0dFaika4+PZqnJvbmI+wTO5q/P3GG0l5bxD137rNB9a89ejLGtMffkip1Hcp0HoXz3IcISaQWvOIHJJ8nzv+3+wAAS6aPltglP0aruCecy+oNu7DrQH9MtjDlifaWQqLmLi6hrSNVc1nGDkSEBZNG4slgYNJL1ny92EBgNp1YPHRESwFrb7pYaqvqu+dRJAlawSN86tKF4eehxlevWBLbJq40FdyTh7Dk7jz3oUIStQREuOtAnePhaxAKuXLdLrz1Ow9K9lDidfjc3y3y86RHn1Ixt4wQJJEQx02Gsoyv4xvKTh4Bv3tfNcu1jSwT8V5jH4AJI1uNSF/0XFWerHZAlShMWyDaZpqXRwv/cn7NJ+lcZBlFJYcyuTvPvc4wvdlsyT0v2VsdCqnR3A3qlSXHIjK3O05Gcc24Mokp8Ng05A4zz90WsgVFEmWZYPq+pByB8NxnL/T3U/jGpbVBM1CqL8dr7tGY/chgay7cLnrVeQyoRmty46nOc29YBCFse/vyl2V0qIZCmkcfpB1QjQopmkLQk1G1LrNJTMZx7mHbZuQT09w9ig38ju+KzsgthKTEleM+FgseigWvElZpZIN6oDRSLubhR/dFPffq3jw89/Aek5xRWk87uM7iZLZDWYN3nvsQIYkPgyW56j3ZSRkKqTkmdSikSWhf+FckLUVBg7qNo2UsiUEsXaD4wt73fvBM9HMx9rKUvOE55xAto+JinYdPVJk4Jqs0T81dNRM3DZQDqocutzvPvf5Qe0w8AvJReY22pG9aXhkKqcvnzlF/iyiGa0Awm8QESMhI2B9E3lQ9d53mXptQSNEoT6K5j2gpYCS3WIPWTsvmAdl1UmnR6gHVgkc4ftZYqR25xLmLWSEz18jJMoHnnlB+5rj2HFptbDjPvUFRTiAfU1JkjMUyJmrL+3/FB840FLLgUSWTvwGIkh/CgHRiGrtkuTcGs0lMxgOqiSXE8pUjjp46Ck/4YwxJXBjEjfPzHaqRH/a0l/iGo9h+1JRReOjF7f4+wjVnzEH/IMNX7no2NsEpK6opIrLXFSCWFTKIlpGUvf9DZ2JUe31WQxpKOHKvM0yf18GEUK44NcnJqswquq6pn/9/fvEE7lmzJTbNXOv5c7uKBQIUY8CxySuGNsmgThyWTBwEYMAg/YAtAhv+37uOx8advb4d+rMM9vOdTRaZImlsItwuXP0bLzwco0aU8KU7nwFRhSyPmNQVK5vHdP7qbRDvxNJq5OqskPH6po1tfq8dcOQ+5FBRZrLnbkbXg2XmrxlqVv6eNVvk7WmO4b1OXe6R+NqZyW8UKv1ZHS0j3y+WrYUsExQf2VbCyIklvw59JYHMIZupbBr5IrMhgOrnEOsqFTwc5a/ZWr12csLMiqyTmGSonuchLLILSBRIiei7RLSZiFZx28YS0Z1E9Kz/dwy378NE9BwRrSGi82tleLNARW7VAVXFcYb1B6RhO+YpPmSGjntspZ5k6C0L7JAlCpPVZBItY665Ww6oSsonh0IGnjtXj/DXzohkmyrb49sCh0LUr/OmS1VyuiwoCJ57NVrm0IXJk/h9ABcI224EcBdjbC6Au/zvIKIFAJYDONI/5htEZLf2WJMjpmUryC142FW5Z97740ej9Si4qpzQSZgiF89d+G4T5y4bQBW/M4luLa2LYJXy1/StR9Zi4iQmf3+5nGy7iWRhqrnL6gpMECdW5Z9ZMaqH51G9Oitk9rqHKxLJnTF2HwBxketLAfzA//wDAK/jtv+UMdbHGHsRwHMAjsvJ1qZEEimryP33qzcZ1R94qFkXJdZGy4gDqoZ1GMkMghdZ3SEvFxKHVnM3k2Vs3T4ZkRt77gayjJkNyTap6g7uNXFAt1b8WK0/ewvifVeLRbiHG9KGQk5gjG0EAP/veH/7FAB8kvJ1/rYYiOgqIlpBRCu2bJHrvIcCVLdgQD5ZJ1IG44a2nnuWB840CsIkFDLMbZ4wOUd8Hc8lFFLRtuYASVsJA6qhLMN57ooKzTpDs2Nk1yd46woWLa+VjF2LqRuilBSsQFWyCM1tNuR95rLbQPpTMsZuZowtY4wt6+npydmMoUNv/yCuuPkBPLlht3R/XMtWyTL5TGKSeYRpYBOPriJWqSxj6GHFZ39Gv8cHVNV11WxAVVLeWJaJSEp27epssNHcz5w/HtefOw+feM2RkTJ5rL7EQwy3lU3gsoX4FvCvf7cIHzx/fiRe/1BDWnLfRESTAMD/u9nfvg7ANK7cVAAb0ps3/PDEKzvx1xe24eO3rUouDANZJmPEXl6au8mao0BlZqMpGdg8yLK8LbK6jDx31EaLlWvu+mMCOWFgkL++0cHBLDaYxrkDlbeIfzx7bhgDrpLEbrrsKPzi3SfaG+ejFhkbxZm+YztacO2Zc2owXjB8kJbcbwNwpf/5SgC/4bYvJ6JWIpoFYC6Ah7KZODyRVUesyjKmM0vl25NCKpUQ3zAMD2tvKSinvIs2Bt52GsQeWiL86bmt+O3fNsj3R4vi1mtOxttOmqlvI6tNMI9zl4ZCWrZfscG0ffPBWbGO5cdNx7Ezs3vEefJuA2QibjiYhEL+BMBfAcwnonVE9E4ANwE4l4ieBXCu/x2MsdUAfg7gSQB3ALiWMWY4X7E5EDyiD63djhVrxXHoOOknD6hmsycMqZTQs83zYErC7aWiRpYRA93TZ+9Tpaz93d9erVStG1AlwsIpo/CB8+Zp20gb525TRzBDNa9JTKY6vU0befOmTJ7LimpuGYcAiZOYGGNXKHadrSj/WQCfzWJUrfDzh1/BEZNGhpM1ao03fOuv0oUSeKhDIaOa+x+f3gQC4czDx0vLq1Ctx+qw2BNoqo2PaCkoyf1rdz0Xbyb1i4VclgmQFOdeKaOnAuvEYSk093CGKq+5h/WJRG3gbYsvNIpzMJLOFLJMVoiZRyPnlbIxT1bXIY5Daij5Q7esxGu+/iejsv2DZXzjnufQG1t7Uo/E6I/YgKq8XODIBR78O76/Am///sNWtgBctIzlcSKZG3vuklWTAnz97ii5mzyG8dzfFagWyA6gWw4uIIJEcrf23CWyTMITFkSoyOSzVLKM2L7Kc7eqq7YDqnkgjzzzzQZ3SRT46UMv43N3rMG37n3e6jjb+FpV6XJOoZBpUwen9ahHtBSMMwcSUep4ZDFRVBLZR/aFZfRt2Gvu8W3Ja7/6A6oGsoyJPbLJXSblpO0Jsz7zQp45hgKocsscynDkrsD+gxWPfV+NF8tQhkL6201DGVUkmTb9gFjetHNoMVjMOgDJGoq1K98uX0NVvT9S1lCWsWUKWfHE3DKyGapZomUMB1QtVJncdez3+It1Hz11tLEtScg7XLMZ4MhdAZN7pbd/EHesejW6MXFSjlnxvOLcyyk1d7Fd0+OLheSl5QJkeR6TFutIinNPKpMGUlkmoY0gXl86QzXNJKbY1N3kY5R11UhzP2lON9bedDHGdLRU6s+h+6gus+dIPoAj9wToSO1Ttz+Jd//wETz28o7c688rzj0kDUtyF+WgJHkokGI27uw1HtTyKHmB7KT2AsS43sCE5AFVO6SRZc46fAIA4JwjJsTqMR2f0dmgjnM3kGWCN4g6EmbalmQrWh3qcOSugMkNvXbrPgDAvr7qoKs9WemjZWTxz7v2my2azdeTObdMwvHnLfBJasEE7WBmrN4ExlLt7miNBnqZzswEzPXZk+d0A6jM3DSBrLokz33B5JFYe9PFWDI9TKya70BjFlmmRp57GluS4EIh43D53BOgo56AOG1G6k0Xnq5mhYzvu+LbDxi3lza3jIgXtuzT7h/T0YLn//kiFDzC1+56NltjBugUyT2F5p70hrFo6qjEUNZovfH68gzNS1NVI2ruqnYy1VGnjmg4wXnuCuhukue37AVjLPSqeU9VJNFNu3uxVzMoq4yW0US5PLkxnrcmMeWv0gIz3PZEchaJ6quxuSyTFiK5x+tW76vV8y/33O1byzNKRVWHXZx7HWWZlE3luXB3s8CRewJE0nzkpR04+4v34ocPvBR67vyNJcoXx//zXbjkq/cb1x/ANv2ACmlDIW2R5tEiSu50VPuTZRl1ncbRPJYnJdfc7eqwrT+pjCq/volZddPaSfrRCm4SUxyO3C3x3OY9AICV63YhyPXEx1zLOHTttv3K+pJCGLOn/M3Hcy96hKM1M3vThu2l7XNKBX0kiY6YTG21n6EqiZbJwO6q9j//hkXGdag8WpvZrjXX3HPoREJbM9fUPHDknoC+gegM1X6f0UtFLyROmwFEEXl47jqvvLe/rG3HFIOMYVZ3R7ZKMiAeDSNGy5iHDZp6d3mQWpp7Q0WqwTl2tZUs6spBcx8GjFlw7B6DI3cFgofiRw++HIlO6fdHOkteNSe4zc1vGt5mkxNGl5t8zabKm4bsDcGG7xkD2jU6dxrvyyQUstZyUq2RRgvOMrfK9HcwkmXCCJTaMqabxFQbOHI3wLZ9feHnIO92seBJpZOsOVwC6JbZEwlDJ908HQy+5sCRHS35LodLMCfvWCZB8XsNnu08qkw1oKqYoWoys9a0ObP0A3Z1pkUu19mrT0c0nODIXQH+FuFfbfv92MJSwZNKJ3nlcNHJMnFyV0eh7/cTn+Xh/4qDmDzSEkBau4iAP7z/NM3+PHTc7HUUC+nrUGd0TF1lte4GCoWMtJnymovL7Dk4cjcCf7/0D/iae4GkkShv+54+c6OxLBNmhYzvE3VcXX+SNv2ADEnhhwGMBywNCqoyCBII8yZ0qes2M0GLIfPcM8gyprDz3Gsty2Sv301iisORuwL8/cZ/HvA996Ln5ZK5UeVzM00IoxjepksuNpDTDFUgwXNPUR8RUrvuSTNSaxkjboNUmrui/er2elGYXB6qTSvZ4NIPxHHIkHvWgbmzvngPbn1sXRgtU+Q899RL2SHdSkxieJ0uoib1Yh0SjCjlr7l3tSW8DRjaHffssyMXWSZTKKTddsDmrcm8zHDgS0fqcaQmdyKaT0SPc/92E9F1RPQJIlrPbb8oT4PTwpbc+HulzCrT76//+RMY8KNlymUWTu03TctbqVeQVBTlymH6AUm2QFHa0SQXG8wpzh2oLH6tQhoiJCJ89JIFWDxtdBazGhpZZqhKduj3W8AscViCPTkh39wyjuUDpCZ3xtgaxthixthiAMcA2A/gVn/3l4N9jLHf5WFoVpiQ22CZ4R/+a0Vs7dMv/mFNpQ5WDYXsL7NcvGLVG4XOcxdv30HGDEIqJZ2EsZUVtBTMbhfzSUIVHf/q02ZbWqKOJKl+b4yHPE9ZRtwv32cYCmnkuddHx+ZtTttWIez4cjCoSZCXLHM2gOcZYy/lVF/uMJFltu3tw51PbsK7f/hohBxuX7kx/HzQJ/eBwXJVlsnA7qojw0y9Us/dTJYpFSgcI8hDltF57mlgEmqnGitIWkO1UR7ybDlP9OcoPcK4Y7Xx3M3qHEoMJwmpXsjraV0O4Cfc9/cS0Uoi+i4RjZEdQERXEdEKIlqxZcuWnMxQw0QWDx7EwXJZeUMf9KNlBsss1YCqabSMLs5dVVZEqeAp88GneWD1sgz32XgiTfpgu8QBVcN6PnbJAuu2bZCK3MNDFB1bLjKGgRkUlG38EVUiwj+eNQfnLpiYvbImQWZyJ6IWAK8F8At/0zcBHAZgMYCNAL4oO44xdjNjbBljbFlPT09WMxIheoB3r9mM3zy+PrItXIlew9aB594/yHLx3FUPcFDnoIScxWfhtsc3SD3colf13EWMGmE+hT1AS9HDdefMxYULc3qA9NKydlsiHxgS0jtPmWVUTocfvvN4XKWQllKlH8hqUMM0Yoa8+o7rz5uPozT5jw415OG5XwjgUcbYJgBgjG1ijA0yn51XmAAAHiVJREFUxsoAvg3guBzayAyRf9/+vYfxvp8+Hi3j/x0sM+W93z/gyzLlci5L4akO1ckyIj7z26ekfQQ/0UqsJk2ET0vBw3XnzMNp8+KdcUQ3tdDcZeX5r9XrIJRJ8NTryV2nzO3GP110hHRfIcMkJtVPn8+ydCayTH4DuPp2qvjUpQtr2tahhDzI/QpwkgwRTeL2vR7AqhzaqAtCb1lDqMGA6gAny8i8axXExyRo6e41m/HIS9Xl+mxkGb4eHny4pujZ7+4dwMvbDxjVHSCQZQZsTliDPHXSRtWF0yUOqxyj+uX1uWUM2zCyw67OPLBg8sg6ttbcyLQSExG1AzgXwNXc5s8R0WJU7s21wr4hg412bSLLDAyWw4cv04Cqf+jb/Zmtwco/gQmmDrbMhqLnYWCQYevePvzq0fWx/Vv39sW26RBEywxIjEpDrirPkIgSR4CTmmsUsrdZpSuAMr49x3Mymh3M8m83rS0O9shE7oyx/QDGCdvemsmiGsGEf4MyMvIKEHrug9UyutmkIuLL7CWFQpqxu8zkUqGSufLd/+8RrHgp/SLeAaqeex5R82rPMCLLCGXFY+VHNU68czENu/tQyzLZYTKgGrztDYPxVAcJDp0ZqiZleH1XcUc/8EIlBp7vAMr8cTnZpdKalfVICpYKHgYZw6u7e+0NkyAg937JIC0pPuuQMFcn4VjR27evox7Iks9dlNLyPCWTzi/03B39DkscOuSuYMnNHPHZyCt8FEqWdUrThELK+EJWT5EbUM0DpYLac88my8i3Z0Gj0FE6WcbX3JWiu+ZY08FsA7tUb015o1E64mbDIUPuKo477p/vwkvb9vllqoWS7rd+juBsVk2KD6jayzKzuztj22T1BLJMXg9Pa1GtuSfhs6+PR0EYTWJKm1umQQgjjSyjfqMxGgY1a8OgTOAQ1fpSujeD2qApyX1Pbz9eFtct1ZDE6Z+/p1LEgrP4cMnguFQDq8IhBw4O+nX5dUoCU3pGtsa2ycoVudWi8kA4oCqJlkkiHl0CrdjDneYtoEH5IYPkXtNQSJt0yw17cR20aDpy377vIM790n047fN3R7a/tH1f4rERzz3hfu4flMgyOWju1/zoEb8uzSCtdJMkWqbgpfKyZSh6FGajTFOnLK6aFCOq0QFVs7bi6Qgag5Ayra+bQnM3bc5oQDXU3GsL13fUBk1H7ks/fWdsAPGFLXvx2q//OfFYnrO27T2oLctzrtWAqnAji8f86bmtkTplPCp7Q1BFy2zZ06dMQWCDEpc07Oip8SyOkQFVydMqm4Zf02e6QQgj3Rqq+mN0u8e0t5i1YSbMJLbn0LhoOnKX4dVdZtEivJf8pTuf0ZblCdY2bDHSZsJiHbI6pc68ZGNff4XV1++0m6wkA59X5uJFk0L93RRScleFQuYgyzQKH2WJ4Y7Pyk0+ZmxHCx78p7MTy5nUVS/P3aE2aHpyPzhQNn7AbNSGCLlnyJmujpZR2yTrEPoG4u757t7+FBbJISYN6+6M6v69/lqtKug893gqgep34wHVWCjk8KWkxAlaCfsnjGwDoF8W0Yjcw7I1Tj8wfH+qhkamSUyNjpe37cdpn78bb1w21ai8TY6YMqve9FUiThMtI+4P6lRr7jLC75cMcualtwPxXO4imR/gvsueVZn2XMt1L4czX+RBdvd/6Ew9uRtcocBpyWNBbh0aZXyk2dDU5P7Mpj0AgN+v3mRU3oYLmUSW0a2IZFJPdHvlr0jQM2/8rbT8QYnnnmX5PxGiDDN/Yhf+8vy28Dv/gB6QePHi0oCAWSikrP40+4cjYrKMxTlOG9uu3W82Q9W+XYfGQVPLMgHpmg5qmWrms7s7Ih2BTh8XEUs/ECsQtWVf34DRG8VBiefen1OaACA6oAoA33zLMfjEa+S50GVrrcpWcZoyeoT0eF52SnsGw/lVXzVDVUR7S/o1bW1yy9Sa24fzb9XIaGrPPSBI08UGTFWZ7s7WyBT8MFrGyrqgUbmHHdg+UGZST1jEqvW7Y9tkhJ8WouY+akQJJxw2Tlr2e28/Fq/u6sWbbn4g3FYUUt92tBTwYT9NrtFU+ISrGwsbHMaEYXI9fnbVCZg+Tu+d69tIRphbJnUrZhjGP1VDo8nJPfgUffDf9YOHFeXN6NnzgMfW7owdpzv+FytewWd++xTGdUZD1RgY/vWOpyW2VAh014F+7DqQbmBUJtWkhUjOQFRH5899xrgOzBjXES0rvD2dv3Ai2nwPP48UtuKlr/nqQTVEt3+PiG9LARiAE2bLO1ZTBJdnfFd8QlykIQzvjvJQRtOQ+4MvbMMHf7kysq2aFiBa9n+e2iytw9RzF6eUm6Qf+NhvVqG3v4yutuglZwz4j/teCL9XZ70yjGmvkPvuAwNmhgnoG0j2+E0hGxClCLnrj1cRFaAn8ENR9/3ymxbjjlWv4ohJQm7zHC8BEeEryxfjmBnSVTABACX/bS3Nql22tjjkj6bR3D/92yfx8vZoyoEguZd52lyzcqIXGx6mOVxFfh8SOiQKNXdglD8hJa3nLtO+00I2IMp740njAjpyz4O0klZqGkpcd85cq/Kj21uw/Ljpyv15LHYOAJcunoKpY9TSzrIZY/BPFx2Of/27Rfk0qEDj/FLNhaYh9/0H415qMKBo+jCYkrvoxVZlGfUx1XQC0e3b9sVnwt6+cgMGywyjfY9p5379bFkV/vPKY1MdJ4MsNwy/Kena6XLLaGG8EpW83LSxI/CZ1w3t0m2XLJqUXMgA9SZBIsJVpx2G0YazXtO3U9PqD1k0DbkfkJB7kJo2y4IXMog3o0mcezVFgb4RAuG9P34MQHUSismAqgwzxrVjbIf+wTTlXFnEkWchy8g0+wAmkksSAcTDBis4YuJIvOWEGYn11xaOvRzqj0zkTkRriehvRPQ4Ea3wt40lojuJ6Fn/r1rUyxEyAgw0ZxNuv//ZLUgbeGeSzz0osyEhFQJ/HkFsuWz2qQxtJQ+XLZ0SfveIEgcWTQcepeTu8eSuv3a6dsRd8vQKevtik8EaiE/ztsU0mdpwQSNJaM2EPDz3Mxljixljy/zvNwK4izE2F8Bd/veaQ+a59/aba+5v/c5DVpOYeAShjLpJQ2l00iD80DTq5T2nz8Hp83rC74RkOcSY3GXJwIjX3PXH68wwGVBNQpYJP/VCmiRiPBwHOtigFtEylwI4w//8AwD3ALihBu1EIPNugynyxpq7IbuL9f304VcwfmRr7tpkqyW5i7HoHlEioXgeAAPVRzagygcNJXegOs89f9ZqJCIMTMmS/jeC5nLcHWqErOTOAPyBiBiA/2CM3QxgAmNsIwAwxjYS0XjZgUR0FYCrAGD6dHVkQBYEhJ+35i5i/c4DuOGWv6U7WIPQczecjCSSOyjZWzQlHPmAqrksk5XXbI9vIG4PO6/M16Chzsqh0ZFVljmZMbYUwIUAriWi00wPZIzdzBhbxhhb1tPTk3xACgSeuym5myYOq5fjZCvLtBS9iBfsKch9/oQurowZYUhDIS0GVMV2eKKKae7854R6rz93HpbNGAPxV2lEzz2viVXOcXcwQSZyZ4xt8P9uBnArgOMAbCKiSQDg/5XPGKoDggFV0xwrOebZygUFz5LcC1Hfjiju66296WK0lex/9iTPPTkKKAp+UNBoKryi+n88ey5++Z6Tcov9rgWCy5Q1u2IjdVgOjY/U5E5EHUTUFXwGcB6AVQBuA3ClX+xKAL/JamRayAZZdTCNQrBJDZwFgWecVpbxCHhha3x5wUHOftMzkabsjWju+uO1KQaEfWk4TGx+8bRKkNYVx9dG8rNB0MXm5rk3cEfm0DjI4rlPAPAnInoCwEMAfssYuwPATQDOJaJnAZzrfx8S7O2zm7bfaJ47UeWfuedeiBClSqN9a4q4b+mAqo3mrl3jVJ0pM+hIk+Pco+1PHNWGtTddjDPnS4d86gqb1MYm9Tg4mCD1gCpj7AUAR0u2bwOQvM5XTugbGMSaV/dI9z2zaW/i8RNHtoVrrqZZJq+WYAwoeZ5xnHtL0Yt0BCXFxKE3HTsdO/b346b/jicsU0EmyxQice76462Iiatr2cyxAIBT5/bga398zuQQLUa2DV06payhkA4ONhj2icM+ffuT+OEDL0v3iblmZIjkDk9B7qUC5Zo3nUeZMZQKZDWgGqzItHjaaBQ1+Vxsw/J0i20ABpq7hSwTOPJzx3di4ZRRAIDjZo3F6PYSdu7PtnTgEx8/L9PxWTCcM1U6DD8M+/QDK9buyHQ8z0llwwy5PI21FvNLzhVrhzEUCx7W7UjupICop56UNMyWZ6Seu1UoZPR4vj6VKSIZ6jok87VW1XU8/ekLzCqxRFWWceTuUD8Me3LPqqREVlRKcbxK+sgDZVbJpvjgi9uVZXjObS16IVHqcrmkgczr5LcNJugy4tEFzr48SC+Pd6e2HLNo8gjOLy9VptnSDzjUBsOe3LPr5NXjzePhq5+1qWwB/O+z5qSyytQePtc2P6CapO/aEmpybhn98WLnkDpL5DBEXnHubhKTgw2GPblnpnbec/e/fOHy2DixEknkPlmxTqgJyix5wY253ISkSihkhQDyJs+k+mw194JOllGsAKTjxnqFp6ZBYFlunnvjnqpDA2H4k3vGO533jgPv8yh/EE/ZJvc5NuVfQEdr+jFrxhj6+vUDAX9/YjWssWJLxbpEz93SFtmAKo/EaBnhe0Rzj81QbS72CnIWJV3DJDjJ3sEGw57cs4KnkeqC2gnHcB1CS4Ln3pFhhfrBMjOawBS0USoQBnwikUXKZCGHJM/ddkC1wM2AykNuaGRvNhiPyCsUsoFP1aGBMOzJvTfBs00CnwmyrJADdNB57tedMzfRsw/Q09UaW1jDdFJVMOO0peiFRJKULiD4KL75qMYIkvRi2zh3XWfR3lJ525k+VlwCbni6rlWnYXja7zA80QTknt8i0NXZkOYPoS5a5riZYxM9+wAjSgWcNrc7ss04m6Xfv7UWCuHqU/KVk6qfVVZ/4Lz5OHLyyNj2JK/TNreMrr4Z49px81uPwRffaDP2kS9x3vKek3KrK43T4OCQFcOe3NMuQRdAloHQxsPSDagSkbHnXmYsIlVU7GFYOCVOtNEyUc89WBRc5hmbdlqXLJoc25ZE7lni3EWNnQCcd+REdLWVYIoRGeQvGY6Zkd8CYixnz72RB48dGgfDmtwZY9nJnZ/EZKi589CRN5F6/79cdlTMDrGfKDPgv95xfKINY9orJJikuUc8dw3RyPqrRHKXqGN/eH81A7R4eCGFp63jxrYMk8l+c+3J+K93HJf6+CQEA6l82GoauElQDjYY1ukH+gbKmQfSmCRaJi/P3SMKV1Pi0VL0cMb8eA77ILKmreSht7+MQcYwtqMFHuk17V+++yT8+fmtKBaqmnswm/NX15yEy77xl9h56U5Rdv5pBlTnTejCd9+2DO0txdigad6hmm0ZPPejp43O0ZI4Znd34KMXH4HXHh1/I0oD57c7mGBYk3seentZ4rknZyCsftZpvUSViUUiBgbLMbIrM4aRvgwxolRAb3858jqvkz1mdndgZneHX3dUc186vSovZFoMO+FYlXlnHT4BALDrQDQnTDQXvJFZWiSlWxhKEBHeders7PXkYIvDoYNhLcsEkkwWL1CWOMzm9bdFIwcQ5LJMmUliu1m1bNDhBFKHdvKO8F2vuUdtU0FG5PsP6tMn2y6zF9XczaCzuZHJPXc4193BAMOb3P3FOLoypHGN5JYJZRn9MXyHoPfcCeM6WzC6Pa61ikcxsDCyJgjPLKfobHSa+0+vOsGqDh4bdvVqj0ki91gSMMMoIlPkPaDaiHCSu4MNhje5+557ZwZyv/q06utyGs1dF+pIVNHkj581VrIz+rWSJKyy8dwjJ+CcI8bj+vPmVXZaeGqDg+o49yMnczNv/XOUVX1A4qVv2HlA226StKKboWoa/XHz3y9T7pONbTQrmm0Gr0NtMKyfCI8Ih/V0YFxHKwBg5jhx0ksyujtb8b/PmuMPWqbR3NVRKWVt5Eq0EcYYLl08BcfNGovrz52H/7zyWEwaVclLw3vFpwqx8CIGDGdDyvZ+6y1LAQD7heUJR7YVceOFh2vrs5VltPYpfoDFmoHPJM/9kkWT8JXli7VlHByaCVnWUJ1GRHcT0VNEtJqI3udv/wQRrSeix/1/F+VnbhRHTBqJuz5wBk6eMw5AOimSqEI0ZQb88enKWt5Zo2WKwsLWsvpisgwDxnS04OdXn4ipY6KdFH9eorYser2yGaqXLp6MD5w7T2r/6fN6MGX0CPz3+07FBQsnAYiT+4/edQIOnyiPtw8mPGXJCpmHH3r+kRO1+7/+5qW4dPGUxHouPmoSPnj+/Bwsqh1cdkgHE2SJlhkA8AHG2KP+QtmPENGd/r4vM8a+kN08MwSTf9JGXQRhgwG5Jz06Ec+9KIks8QAMIlweTybLizq6znTeK05KRBZ67lyjX1m+RNJ+5e/o9hL+fONZkX3iwuK6vu5bbzkGp37ubuvUy/zbjHjo9YqOCADec8ZhePSl+AIt3Z2tOG1eD+57ZouVHSL+/X8tzXR8PdCMsszhE7vwuiXJna+DObKsoboRwEb/8x4iegrAkPw6Jd8LPHbm2NjSekkx4kD8ld4jwpUnzsBtT2zAjoRl3VolnnvQWQTpes08d7WR/K72BPnhtUdPxrfufT7Rk9Vhv0WIaTVHjVm5ALpVlU6fF58DEOCGC9TyULPP3Gxmf/2O605LLuRghVw0dyKaCWAJgAf9Te8lopVE9F0iks7jJqKriGgFEa3YsiWbtxV4qd1dLVh708ViO9pjT57TjZ6u1sg2jwifvHQh7rz+9MS2ByWE4oXkXlbaIG4yTRKW5LkvmDwSa2+6GIf1dGrLBa/2Mj58wzFTK235HYnJhKfklZjErJDNTFW1RZP3YQ45ITO5E1EngFsAXMcY2w3gmwAOA7AYFc/+i7LjGGM3M8aWMcaW9fSoPTUTBPrtoGShah2F/PnGs3BYTyd6OqPkTv5VMdHe9/TGI0tes7gyE3HR1NF+PfHjRLIz9TrziufWndrp83qw9qaLMcdfCKRPs0D3mPZKJsu3nzxT2554DaLRPPmwVbOTnks/4GCDTORORCVUiP1HjLFfAQBjbBNjbJAxVgbwbQC1S9rhI9DcA72ZzxMiex6CuPOJI9sAAN0Szx1Qdwy85ikj91PndGPtTRdjlj9rVNpJSCYxmUCUZbKnX9C05Xck+/vUMs2IlgLW3nQxrj79MG07sXzu3JhAXqQc6P4/fGdyPp7hjGbvxBzyQZZoGQLwHQBPMca+xG2fxBV7PYBV6c0zQ+i5C5N/AHlkwUcvXoBnP3thKA1M8ElerM/EUTplTjU08XNvWAQAmDM+KonIVuCJyzJmT2x7hpWdIu37f3WDc0FHkjQ71aa9AId1y2WjpFBPHYJL2KwO7rkLKqkcZvd0DLElDsMBWTz3kwG8FcBZQtjj54job0S0EsCZAN6fh6E6FH0vMPDceZ6UPeiTR7dFQhhHjSjh/edUIzTaSoHWXD149SfPDz8H9X/775fhsqXVMeQ3LpuGVZ88P7KuKSCXZUQuf41hUql2QZYROxJTmBDg2UdUyGRabNGMbO2t+uT5mM7NSQguxZGTR+J7bzs2dRtBR9Wk3I7lx1bur9kJ4ykODkC2aJk/Qf4c/S69Oekw3yfTxdMqMzB5b3TqmBF4fsu+SPlpY+JkNbazoh13cZ4xT0iygczO1mJMbuiUlJMNHooa+2detzBWRoYiJ2c8/JFzYoPBpgg6BT6xmIgrjpuGcxdMSN0GD/46ya4RUBlPkE34MkV4SZuU3YlIee0cHEQM6xmqAZbNHIv7Pngm3rhsGoBqwq254ztxub9tyugRYflJo9pidbT509dHcjm3VQOqtpLnQsmC26LnbkpqgU0LJo3MRLrHzBiL+z90Jt507DRlGSLKhdjrheCSuuXsHByGecpfHrLX/Olj23HVqbNx1uHj8eU7n8H6nQdw44WHS4k0kGL4JGQqilg0ZRQeenE7ujtbFCWiuPyYqVg0dRRKBQ9nf/FejCgVUEqZC6XgEe7/0JkY02HWtg55yC15IC+tPMzqmdEeB4dmQNOQO49q6t7KYOa8CV2hNyeTZIBqqtwZXCeh8gBvuPBwvHbx5Ji2rgIRhdP37/3gGehsLaKztYjf/eOpuOir95udFGdTo5ByXqiScjZarnYSjt4dHJpClhERSq+SBOaqqJSAWE6YPa66TcERpYIXxrDbYsa4Dozz4+oXSBaiTsJwn/wzXdIx5RXZF4SejhzRlD6Lg4MVmvIpkL2eB5q7Kvf7a46ejLZSAef54WY8GolQR2ZIbzzU+PnVJ+IwSRhfMEg4ZcyI2D4bfOrShbho0SRlkjMHh0MJw5cpNGhvqZwWPxj4gfPm4cjJI5V5Swoe4YKF8nwsM1KkEjbFr645KVxeLwlfWb4Yx3NvFsMNx8ny2qMy4Pzvb14qXVfWBiNaCjhz/vhMdTg4NAuaktxPnduNmy47KpLitbVYMEr5yqOtVMC/v3kpls1UhwsCwK3XnJRaWtCFIoqwtX844eJFk5ILOTg4GKMpyZ2IsPy46bnUxZPOD995vHRRiCUWBO3g4OBQDzQludcKp2SYGu/g4OBQTzhyHwb4/957Ch57Jb5AhYODg4MKjtyHAY6aOgpHTY3PcnVwcHBQoSnj3B0cHBwOdThyd3BwcGhCOHJ3cHBwaEI4cndwcHBoQjhyd3BwcGhCOHJ3cHBwaEI4cndwcHBoQjhyd3BwcGhCkLiW55AYQbQFwEsZqugGsDUnc2oBZ182NLp9QOPb6OzLjka0cQZjTJpOtSHIPSuIaAVjbNlQ26GCsy8bGt0+oPFtdPZlx3CwkYeTZRwcHByaEI7cHRwcHJoQzULuNw+1AQlw9mVDo9sHNL6Nzr7sGA42hmgKzd3BwcHBIYpm8dwdHBwcHDg4cndwcHBoQgxrcieiC4hoDRE9R0Q3DpEN3yWizUS0its2lojuJKJn/b9juH0f9u1dQ0Tn18G+aUR0NxE9RUSrieh9DWhjGxE9RERP+DZ+stFs9NssENFjRHR7o9lHRGuJ6G9E9DgRrWg0+/w2RxPRL4noaf9+PLFRbCSi+f61C/7tJqLrGsW+VGCMDct/AAoAngcwG0ALgCcALBgCO04DsBTAKm7b5wDc6H++EcC/+p8X+Ha2Apjl21+osX2TACz1P3cBeMa3o5FsJACd/ucSgAcBnNBINvrtXg/gxwBub8DfeS2AbmFbw9jnt/sDAO/yP7cAGN1oNvptFwC8CmBGI9pnfB5DbUCGH+BEAL/nvn8YwIeHyJaZiJL7GgCT/M+TAKyR2Qjg9wBOrLOtvwFwbqPaCKAdwKMAjm8kGwFMBXAXgLM4cm8k+2Tk3kj2jQTwIvwgjka0kWvrPAB/blT7TP8NZ1lmCoBXuO/r/G2NgAmMsY0A4P8d728fUpuJaCaAJah4xg1loy95PA5gM4A7GWONZuO/AfgQgDK3rZHsYwD+QESPENFVDWjfbABbAHzPl7b+k4g6GszGAMsB/MT/3Ij2GWE4kztJtjV6XOeQ2UxEnQBuAXAdY2y3rqhkW81tZIwNMsYWo+IhH0dECzXF62ojEV0CYDNj7BHTQyTban0NT2aMLQVwIYBrieg0TdmhsK+Iinz5TcbYEgD7UJE5VBiS+5CIWgC8FsAvkopKtjUU/wxncl8HYBr3fSqADUNki4hNRDQJAPy/m/3tQ2IzEZVQIfYfMcZ+1Yg2BmCM7QRwD4ALGsjGkwG8lojWAvgpgLOI6IcNZB8YYxv8v5sB3ArguEayz29znf9GBgC/RIXsG8lGoNI5PsoY2+R/bzT7jDGcyf1hAHOJaJbf2y4HcNsQ2xTgNgBX+p+vREXnDrYvJ6JWIpoFYC6Ah2ppCBERgO8AeIox9qUGtbGHiEb7n0cAOAfA041iI2Psw4yxqYyxmajcZ39kjL2lUewjog4i6go+o6IZr2oU+wCAMfYqgFeIaL6/6WwATzaSjT6uQFWSCexoJPvMMdSif8aBj4tQif54HsBHhsiGnwDYCKAfld78nQDGoTL49qz/dyxX/iO+vWsAXFgH+05B5XVxJYDH/X8XNZiNiwA85tu4CsD/9bc3jI1cu2egOqDaEPahomc/4f9bHTwLjWIf1+ZiACv83/nXAMY0ko2oDOZvAzCK29Yw9tn+c+kHHBwcHJoQw1mWcXBwcHBQwJG7g4ODQxPCkbuDg4NDE8KRu4ODg0MTwpG7g4ODQxPCkbuDg4NDE8KRu4ODg0MT4v8Hs3tn0K9Fw8YAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def play_episode(env, agent, max_episode_steps=None, mode=None, render=False):\n",
    "    observation, reward, done = env.reset(), 0., False\n",
    "    agent.reset(mode=mode)\n",
    "    episode_reward, elapsed_steps = 0., 0\n",
    "    while True:\n",
    "        action = agent.step(observation, reward, done)\n",
    "        if render:\n",
    "            env.render()\n",
    "        if done:\n",
    "            break\n",
    "        observation, reward, done, _ = env.step(action)\n",
    "        episode_reward += reward\n",
    "        elapsed_steps += 1\n",
    "        if max_episode_steps and elapsed_steps >= max_episode_steps:\n",
    "            break\n",
    "    agent.close()\n",
    "    return episode_reward, elapsed_steps\n",
    "\n",
    "\n",
    "logging.info('==== train ====')\n",
    "episode_rewards = []\n",
    "for episode in itertools.count():\n",
    "    episode_reward, elapsed_steps = play_episode(env.unwrapped, agent,\n",
    "            max_episode_steps=env._max_episode_steps, mode='train')\n",
    "    episode_rewards.append(episode_reward)\n",
    "    logging.debug('train episode %d: reward = %.2f, steps = %d',\n",
    "            episode, episode_reward, elapsed_steps)\n",
    "    if np.mean(episode_rewards[-10:]) > 195:\n",
    "        break\n",
    "plt.plot(episode_rewards)\n",
    "\n",
    "\n",
    "logging.info('==== test ====')\n",
    "episode_rewards = []\n",
    "for episode in range(100):\n",
    "    episode_reward, elapsed_steps = play_episode(env, agent)\n",
    "    episode_rewards.append(episode_reward)\n",
    "    logging.debug('test episode %d: reward = %.2f, steps = %d',\n",
    "            episode, episode_reward, elapsed_steps)\n",
    "logging.info('average episode reward = %.2f ± %.2f',\n",
    "        np.mean(episode_rewards), np.std(episode_rewards))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
